From 41cffbb2f229bb4c1b2d03d0813e4be1f9931e92 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 1 Aug 2022 20:18:28 +0200 Subject: [PATCH 1/4] cmake: enable CMAKE_COLOR_DIAGNOSTICS by default Related to https://github.com/espressif/esp-idf/issues/4162 Setting this option informs CMake that it should pass -fcolor-diagnostics flag to the compiler. (Colorized build system output, like from GNU Make, is produced even without this flag.) Note that if the build is done using Ninja and the build output is redirected (not a TTY), Ninja will still strip the escape codes from the output. For the case of idf.py, this is handled in the next commit. --- tools/cmake/project.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/cmake/project.cmake b/tools/cmake/project.cmake index b098926bc0..99230900ee 100644 --- a/tools/cmake/project.cmake +++ b/tools/cmake/project.cmake @@ -344,6 +344,12 @@ macro(project project_name) # Generate compile_commands.json (needs to come after project call). set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + # If CMAKE_COLOR_DIAGNOSTICS not set in project CMakeLists.txt or in the environment, + # enable it by default. + if(NOT DEFINED CMAKE_COLOR_DIAGNOSTICS AND NOT DEFINED ENV{CMAKE_COLOR_DIAGNOSTICS}) + set(CMAKE_COLOR_DIAGNOSTICS ON) + endif() + # Since components can import third-party libraries, the original definition of project() should be restored # before the call to add components to the build. function(project) From 12abe1f3163f55cd41594469629a463a2abe580a Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 1 Aug 2022 20:24:02 +0200 Subject: [PATCH 2/4] tools: idf.py: enable CLICOLOR_FORCE for interactive builds If stdout is a TTY (meaning that the output is not redirected), tell the build tool (GNU Make or Ninja) to enable colorized output. GNU Make and Ninja also check if their stdout is redirected and strip color escape sequences in that case. CLICOLOR_FORCE environment variable overrides this behavior. With this change, if the compiler was launched with the -fcolor-diagnostics flag and idf.py output is not redirected, the final output in the terminal will be colorized. (-fcolor-diagnostics is handled at CMake level by the previous commit) --- tools/idf_py_actions/constants.py | 6 +----- tools/idf_py_actions/tools.py | 8 +++++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/idf_py_actions/constants.py b/tools/idf_py_actions/constants.py index 18ebe3e7c2..19e9e99587 100644 --- a/tools/idf_py_actions/constants.py +++ b/tools/idf_py_actions/constants.py @@ -12,7 +12,6 @@ GENERATORS: Dict[str, Union[str, Dict, list]] = collections.OrderedDict([ # - dry_run: command to run in dry run mode # - verbose_flag: verbose flag # - force_progression: one liner status of the progress - # - envvar: environment variables ('Ninja', { 'command': ['ninja'], 'version': ['ninja', '--version'], @@ -20,7 +19,6 @@ GENERATORS: Dict[str, Union[str, Dict, list]] = collections.OrderedDict([ 'verbose_flag': '-v', # as opposed to printing the status updates each in a in new line 'force_progression': True, - 'envvar': {} }), ]) @@ -30,9 +28,7 @@ if os.name != 'nt': 'version': [MAKE_CMD, '--version'], 'dry_run': [MAKE_CMD, '-n'], 'verbose_flag': 'VERBOSE=1', - 'force_progression': False, - # CLICOLOR_FORCE if set forcing make to print ANSI escape sequence - 'envvar': {'CLICOLOR_FORCE': '1'}} + 'force_progression': False} URL_TO_DOC = 'https://docs.espressif.com/projects/esp-idf' diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index 14fddffeb9..f211d53839 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -300,11 +300,17 @@ def run_target(target_name: str, args: 'PropertyDict', env: Optional[Dict]=None, env = {} generator_cmd = GENERATORS[args.generator]['command'] - env.update(GENERATORS[args.generator]['envvar']) if args.verbose: generator_cmd += [GENERATORS[args.generator]['verbose_flag']] + # By default, GNU Make and Ninja strip away color escape sequences when they see that their stdout is redirected. + # If idf.py's stdout is not redirected, the final output is a TTY, so we can tell Make/Ninja to disable stripping + # of color escape sequences. (Requires Ninja v1.9.0 or later.) + if sys.stdout.isatty(): + if 'CLICOLOR_FORCE' not in env: + env['CLICOLOR_FORCE'] = '1' + RunTool(generator_cmd[0], generator_cmd + [target_name], args.build_dir, env, custom_error_handler, hints=not args.no_hints, force_progression=force_progression, interactive=interactive)() From 712743d777968c4a24a27490da633c74a5194c1b Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 1 Aug 2022 20:33:17 +0200 Subject: [PATCH 3/4] idf.py: ensure that build log is always sanitized from color sequences The actual output from the build tool (CMake/Ninja) may or may not contain color escape codes, depending on various factors. The output written to the log file should never include color escape codes, though. This is because color escape codes in files are usually not rendered as "color" in editors, and complicate reading. Also escape codes would break the regular expressions used to display hints for compilation errors. --- tools/idf_py_actions/tools.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index f211d53839..81b5b2706b 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -233,12 +233,6 @@ class RunTool: ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') return ansi_escape.sub('', text) - def prepare_for_print(out: str) -> str: - if not output_stream.isatty(): - # delete escape sequence if we printing in environments where ANSI coloring is disabled - return delete_ansi_escape(out) - return out - def print_progression(output: str) -> None: # Print a new line on top of the previous line sys.stdout.write('\x1b[K') @@ -275,8 +269,15 @@ class RunTool: output = await read_stream() if not output: break - output = prepare_for_print(output) - output_file.write(output) + output_noescape = delete_ansi_escape(output) + # Always remove escape sequences when writing the build log. + output_file.write(output_noescape) + # If idf.py output is redirected and the output stream is not a TTY, + # strip the escape sequences as well. + # (There shouldn't be any, but just in case.) + if not output_stream.isatty(): + output = output_noescape + if self.force_progression and output[0] == '[' and '-v' not in self.args and output_stream.isatty(): # print output in progression way but only the progression related (that started with '[') and if verbose flag is not set print_progression(output) From 990681764e799b9369cc54b4fcc3e2432ac30171 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Tue, 9 Aug 2022 02:37:59 +0200 Subject: [PATCH 4/4] tools: cmake: upgrade from 3.23.1 to 3.24.0 --- tools/tools.json | 49 ++++++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/tools/tools.json b/tools/tools.json index d09e32b07b..1ef44362c3 100644 --- a/tools/tools.json +++ b/tools/tools.json @@ -689,47 +689,42 @@ "version_regex": "cmake version ([0-9.]+)", "versions": [ { - "linux-amd64": { - "sha256": "f3c654b2e226b9d43369e0bd8487c51618d4dbe5a1af929dd32af7e6ca432d60", - "size": 45998644, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz" - }, "linux-arm64": { - "sha256": "74062efddeb935bce3d33694a4db534cef9a650f77a9a153a9f217d9dc385c75", - "size": 47458032, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-aarch64.tar.gz" + "sha256": "50c3b8e9d3a3cde850dd1ea143df9d1ae546cbc5e74dc6d223eefc1979189651", + "size": 48478082, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-linux-aarch64.tar.gz" }, "linux-armel": { - "sha256": "aa6079237e16cc3b389479b2f7279d07e57f6aedad520e2b3014ef97fb906466", - "size": 19330381, - "url": "https://dl.espressif.com/dl/cmake/cmake-3.23.1-Linux-armv7l.tar.gz" + "sha256": "7dc787ef968dfef92491a4f191b8739ff70f8a649608b811c7a737b52481beb0", + "size": 19811327, + "url": "https://dl.espressif.com/dl/cmake/cmake-3.24.0-Linux-armv7l.tar.gz" }, "linux-armhf": { - "sha256": "aa6079237e16cc3b389479b2f7279d07e57f6aedad520e2b3014ef97fb906466", - "size": 19330381, - "url": "https://dl.espressif.com/dl/cmake/cmake-3.23.1-Linux-armv7l.tar.gz" + "sha256": "7dc787ef968dfef92491a4f191b8739ff70f8a649608b811c7a737b52481beb0", + "size": 19811327, + "url": "https://dl.espressif.com/dl/cmake/cmake-3.24.0-Linux-armv7l.tar.gz" }, "macos": { - "sha256": "f794ed92ccb4e9b6619a77328f313497d7decf8fb7e047ba35a348b838e0e1e2", - "size": 70988516, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-macos-universal.tar.gz" + "sha256": "3e0cca74a56d9027dabb845a5a26e42ef8e8b33beb1655d6a724187a345145e4", + "size": 72801419, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-macos-universal.tar.gz" }, "macos-arm64": { - "sha256": "f794ed92ccb4e9b6619a77328f313497d7decf8fb7e047ba35a348b838e0e1e2", - "size": 70988516, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-macos-universal.tar.gz" + "sha256": "3e0cca74a56d9027dabb845a5a26e42ef8e8b33beb1655d6a724187a345145e4", + "size": 72801419, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-macos-universal.tar.gz" }, - "name": "3.23.1", + "name": "3.24.0", "status": "recommended", "win32": { - "sha256": "9b509cc4eb7191dc128cfa3f2170036f9cbc7d9d5f93ff7fafc5b2d77b3b40dc", - "size": 39070972, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-windows-x86_64.zip" + "sha256": "b1ad8c2dbf0778e3efcc9fd61cd4a962e5c1af40aabdebee3d5074bcff2e103c", + "size": 40212531, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-windows-x86_64.zip" }, "win64": { - "sha256": "9b509cc4eb7191dc128cfa3f2170036f9cbc7d9d5f93ff7fafc5b2d77b3b40dc", - "size": 39070972, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-windows-x86_64.zip" + "sha256": "b1ad8c2dbf0778e3efcc9fd61cd4a962e5c1af40aabdebee3d5074bcff2e103c", + "size": 40212531, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-windows-x86_64.zip" } }, {