From e40b83c16f7eaf72fb1218aa9b068d6327d86ff3 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Tue, 9 Aug 2022 02:37:59 +0200 Subject: [PATCH 1/4] tools: cmake: upgrade from 3.23.1 to 3.24.0 --- tools/tools.json | 50 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tools/tools.json b/tools/tools.json index 7fc16d8ddb..7d2182d3fa 100644 --- a/tools/tools.json +++ b/tools/tools.json @@ -689,46 +689,46 @@ "versions": [ { "linux-amd64": { - "sha256": "f3c654b2e226b9d43369e0bd8487c51618d4dbe5a1af929dd32af7e6ca432d60", - "size": 45998644, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz" + "sha256": "726f88e6598523911e4bce9b059dc20b851aa77f97e4cc5573f4e42775a5c16f", + "size": 47042675, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-linux-x86_64.tar.gz" }, "linux-arm64": { - "sha256": "74062efddeb935bce3d33694a4db534cef9a650f77a9a153a9f217d9dc385c75", - "size": 47458032, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-aarch64.tar.gz" + "sha256": "50c3b8e9d3a3cde850dd1ea143df9d1ae546cbc5e74dc6d223eefc1979189651", + "size": 48478082, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-linux-aarch64.tar.gz" }, "linux-armel": { - "sha256": "aa6079237e16cc3b389479b2f7279d07e57f6aedad520e2b3014ef97fb906466", - "size": 19330381, - "url": "https://dl.espressif.com/dl/cmake/cmake-3.23.1-Linux-armv7l.tar.gz" + "sha256": "7dc787ef968dfef92491a4f191b8739ff70f8a649608b811c7a737b52481beb0", + "size": 19811327, + "url": "https://dl.espressif.com/dl/cmake/cmake-3.24.0-Linux-armv7l.tar.gz" }, "linux-armhf": { - "sha256": "aa6079237e16cc3b389479b2f7279d07e57f6aedad520e2b3014ef97fb906466", - "size": 19330381, - "url": "https://dl.espressif.com/dl/cmake/cmake-3.23.1-Linux-armv7l.tar.gz" + "sha256": "7dc787ef968dfef92491a4f191b8739ff70f8a649608b811c7a737b52481beb0", + "size": 19811327, + "url": "https://dl.espressif.com/dl/cmake/cmake-3.24.0-Linux-armv7l.tar.gz" }, "macos": { - "sha256": "f794ed92ccb4e9b6619a77328f313497d7decf8fb7e047ba35a348b838e0e1e2", - "size": 70988516, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-macos-universal.tar.gz" + "sha256": "3e0cca74a56d9027dabb845a5a26e42ef8e8b33beb1655d6a724187a345145e4", + "size": 72801419, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-macos-universal.tar.gz" }, "macos-arm64": { - "sha256": "f794ed92ccb4e9b6619a77328f313497d7decf8fb7e047ba35a348b838e0e1e2", - "size": 70988516, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-macos-universal.tar.gz" + "sha256": "3e0cca74a56d9027dabb845a5a26e42ef8e8b33beb1655d6a724187a345145e4", + "size": 72801419, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-macos-universal.tar.gz" }, - "name": "3.23.1", + "name": "3.24.0", "status": "recommended", "win32": { - "sha256": "9b509cc4eb7191dc128cfa3f2170036f9cbc7d9d5f93ff7fafc5b2d77b3b40dc", - "size": 39070972, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-windows-x86_64.zip" + "sha256": "b1ad8c2dbf0778e3efcc9fd61cd4a962e5c1af40aabdebee3d5074bcff2e103c", + "size": 40212531, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-windows-x86_64.zip" }, "win64": { - "sha256": "9b509cc4eb7191dc128cfa3f2170036f9cbc7d9d5f93ff7fafc5b2d77b3b40dc", - "size": 39070972, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-windows-x86_64.zip" + "sha256": "b1ad8c2dbf0778e3efcc9fd61cd4a962e5c1af40aabdebee3d5074bcff2e103c", + "size": 40212531, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-windows-x86_64.zip" } }, { From 1ffbee27e7bfd32ee69bf79650a8137b5c793cdb Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 1 Aug 2022 20:18:28 +0200 Subject: [PATCH 2/4] cmake: enable CMAKE_COLOR_DIAGNOSTICS by default Related to https://github.com/espressif/esp-idf/issues/4162 Setting this option informs CMake that it should pass -fcolor-diagnostics flag to the compiler. (Colorized build system output, like from GNU Make, is produced even without this flag.) Note that if the build is done using Ninja and the build output is redirected (not a TTY), Ninja will still strip the escape codes from the output. For the case of idf.py, this is handled in the next commit. --- tools/cmake/project.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/cmake/project.cmake b/tools/cmake/project.cmake index a051cb6629..0978d3930d 100644 --- a/tools/cmake/project.cmake +++ b/tools/cmake/project.cmake @@ -344,6 +344,12 @@ macro(project project_name) # Generate compile_commands.json (needs to come after project call). set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + # If CMAKE_COLOR_DIAGNOSTICS not set in project CMakeLists.txt or in the environment, + # enable it by default. + if(NOT DEFINED CMAKE_COLOR_DIAGNOSTICS AND NOT DEFINED ENV{CMAKE_COLOR_DIAGNOSTICS}) + set(CMAKE_COLOR_DIAGNOSTICS ON) + endif() + # Since components can import third-party libraries, the original definition of project() should be restored # before the call to add components to the build. function(project) From 22093dda216b732cbfe4324ecd74ed839d8b483d Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 1 Aug 2022 20:24:02 +0200 Subject: [PATCH 3/4] tools: idf.py: enable CLICOLOR_FORCE for interactive builds If stdout is a TTY (meaning that the output is not redirected), tell the build tool (GNU Make or Ninja) to enable colorized output. GNU Make and Ninja also check if their stdout is redirected and strip color escape sequences in that case. CLICOLOR_FORCE environment variable overrides this behavior. With this change, if the compiler was launched with the -fcolor-diagnostics flag and idf.py output is not redirected, the final output in the terminal will be colorized. (-fcolor-diagnostics is handled at CMake level by the previous commit) --- tools/idf_py_actions/constants.py | 6 +----- tools/idf_py_actions/tools.py | 8 +++++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/idf_py_actions/constants.py b/tools/idf_py_actions/constants.py index bd0f1f291a..18ae271592 100644 --- a/tools/idf_py_actions/constants.py +++ b/tools/idf_py_actions/constants.py @@ -12,7 +12,6 @@ GENERATORS: Dict[str, Union[str, Dict, list]] = collections.OrderedDict([ # - dry_run: command to run in dry run mode # - verbose_flag: verbose flag # - force_progression: one liner status of the progress - # - envvar: environment variables ('Ninja', { 'command': ['ninja'], 'version': ['ninja', '--version'], @@ -20,7 +19,6 @@ GENERATORS: Dict[str, Union[str, Dict, list]] = collections.OrderedDict([ 'verbose_flag': '-v', # as opposed to printing the status updates each in a in new line 'force_progression': True, - 'envvar': {} }), ]) @@ -30,9 +28,7 @@ if os.name != 'nt': 'version': [MAKE_CMD, '--version'], 'dry_run': [MAKE_CMD, '-n'], 'verbose_flag': 'VERBOSE=1', - 'force_progression': False, - # CLICOLOR_FORCE if set forcing make to print ANSI escape sequence - 'envvar': {'CLICOLOR_FORCE': '1'}} + 'force_progression': False} URL_TO_DOC = 'https://docs.espressif.com/projects/esp-idf' diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index 8a009568fe..13f82f8d18 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -304,11 +304,17 @@ def run_target(target_name: str, args: 'PropertyDict', env: Optional[Dict]=None, env = {} generator_cmd = GENERATORS[args.generator]['command'] - env.update(GENERATORS[args.generator]['envvar']) if args.verbose: generator_cmd += [GENERATORS[args.generator]['verbose_flag']] + # By default, GNU Make and Ninja strip away color escape sequences when they see that their stdout is redirected. + # If idf.py's stdout is not redirected, the final output is a TTY, so we can tell Make/Ninja to disable stripping + # of color escape sequences. (Requires Ninja v1.9.0 or later.) + if sys.stdout.isatty(): + if 'CLICOLOR_FORCE' not in env: + env['CLICOLOR_FORCE'] = '1' + RunTool(generator_cmd[0], generator_cmd + [target_name], args.build_dir, env, custom_error_handler, hints=not args.no_hints, force_progression=force_progression, interactive=interactive)() From c9130e48598e3fd61545aae469158728984a411a Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 1 Aug 2022 20:33:17 +0200 Subject: [PATCH 4/4] idf.py: ensure that build log is always sanitized from color sequences The actual output from the build tool (CMake/Ninja) may or may not contain color escape codes, depending on various factors. The output written to the log file should never include color escape codes, though. This is because color escape codes in files are usually not rendered as "color" in editors, and complicate reading. Also escape codes would break the regular expressions used to display hints for compilation errors. --- tools/idf_py_actions/tools.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index 13f82f8d18..92e9ef0466 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -233,12 +233,6 @@ class RunTool: ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') return ansi_escape.sub('', text) - def prepare_for_print(out: str) -> str: - if not output_stream.isatty(): - # delete escape sequence if we printing in environments where ANSI coloring is disabled - return delete_ansi_escape(out) - return out - def print_progression(output: str) -> None: # Print a new line on top of the previous line sys.stdout.write('\x1b[K') @@ -279,8 +273,15 @@ class RunTool: output = await read_stream() if not output: break - output = prepare_for_print(output) - output_file.write(output) + output_noescape = delete_ansi_escape(output) + # Always remove escape sequences when writing the build log. + output_file.write(output_noescape) + # If idf.py output is redirected and the output stream is not a TTY, + # strip the escape sequences as well. + # (There shouldn't be any, but just in case.) + if not output_stream.isatty(): + output = output_noescape + if self.force_progression and output[0] == '[' and '-v' not in self.args and output_stream.isatty(): # print output in progression way but only the progression related (that started with '[') and if verbose flag is not set print_progression(output)