From 56e626335061a64809b284287798bb9044c8fd1d Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 1 Aug 2022 20:18:28 +0200 Subject: [PATCH 1/4] cmake: enable CMAKE_COLOR_DIAGNOSTICS by default Related to https://github.com/espressif/esp-idf/issues/4162 Setting this option informs CMake that it should pass -fcolor-diagnostics flag to the compiler. (Colorized build system output, like from GNU Make, is produced even without this flag.) Note that if the build is done using Ninja and the build output is redirected (not a TTY), Ninja will still strip the escape codes from the output. For the case of idf.py, this is handled in the next commit. --- tools/cmake/project.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/cmake/project.cmake b/tools/cmake/project.cmake index b098926bc0..99230900ee 100644 --- a/tools/cmake/project.cmake +++ b/tools/cmake/project.cmake @@ -344,6 +344,12 @@ macro(project project_name) # Generate compile_commands.json (needs to come after project call). set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + # If CMAKE_COLOR_DIAGNOSTICS not set in project CMakeLists.txt or in the environment, + # enable it by default. + if(NOT DEFINED CMAKE_COLOR_DIAGNOSTICS AND NOT DEFINED ENV{CMAKE_COLOR_DIAGNOSTICS}) + set(CMAKE_COLOR_DIAGNOSTICS ON) + endif() + # Since components can import third-party libraries, the original definition of project() should be restored # before the call to add components to the build. function(project) From d04b5540655d1b78f7690e2bd0c5c041814cec11 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 1 Aug 2022 20:24:02 +0200 Subject: [PATCH 2/4] tools: idf.py: enable CLICOLOR_FORCE for interactive builds If stdout is a TTY (meaning that the output is not redirected), tell the build tool (GNU Make or Ninja) to enable colorized output. GNU Make and Ninja also check if their stdout is redirected and strip color escape sequences in that case. CLICOLOR_FORCE environment variable overrides this behavior. With this change, if the compiler was launched with the -fcolor-diagnostics flag and idf.py output is not redirected, the final output in the terminal will be colorized. (-fcolor-diagnostics is handled at CMake level by the previous commit) --- tools/idf_py_actions/constants.py | 6 +----- tools/idf_py_actions/tools.py | 8 +++++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/idf_py_actions/constants.py b/tools/idf_py_actions/constants.py index 18ebe3e7c2..19e9e99587 100644 --- a/tools/idf_py_actions/constants.py +++ b/tools/idf_py_actions/constants.py @@ -12,7 +12,6 @@ GENERATORS: Dict[str, Union[str, Dict, list]] = collections.OrderedDict([ # - dry_run: command to run in dry run mode # - verbose_flag: verbose flag # - force_progression: one liner status of the progress - # - envvar: environment variables ('Ninja', { 'command': ['ninja'], 'version': ['ninja', '--version'], @@ -20,7 +19,6 @@ GENERATORS: Dict[str, Union[str, Dict, list]] = collections.OrderedDict([ 'verbose_flag': '-v', # as opposed to printing the status updates each in a in new line 'force_progression': True, - 'envvar': {} }), ]) @@ -30,9 +28,7 @@ if os.name != 'nt': 'version': [MAKE_CMD, '--version'], 'dry_run': [MAKE_CMD, '-n'], 'verbose_flag': 'VERBOSE=1', - 'force_progression': False, - # CLICOLOR_FORCE if set forcing make to print ANSI escape sequence - 'envvar': {'CLICOLOR_FORCE': '1'}} + 'force_progression': False} URL_TO_DOC = 'https://docs.espressif.com/projects/esp-idf' diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index 2eecd1d2b1..854ca48a10 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -300,11 +300,17 @@ def run_target(target_name: str, args: 'PropertyDict', env: Optional[Dict]=None, env = {} generator_cmd = GENERATORS[args.generator]['command'] - env.update(GENERATORS[args.generator]['envvar']) if args.verbose: generator_cmd += [GENERATORS[args.generator]['verbose_flag']] + # By default, GNU Make and Ninja strip away color escape sequences when they see that their stdout is redirected. + # If idf.py's stdout is not redirected, the final output is a TTY, so we can tell Make/Ninja to disable stripping + # of color escape sequences. (Requires Ninja v1.9.0 or later.) + if sys.stdout.isatty(): + if 'CLICOLOR_FORCE' not in env: + env['CLICOLOR_FORCE'] = '1' + RunTool(generator_cmd[0], generator_cmd + [target_name], args.build_dir, env, custom_error_handler, hints=not args.no_hints, force_progression=force_progression, interactive=interactive)() From 13881a3832af4d0321e7589eaf5f0432244e9f3a Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 1 Aug 2022 20:33:17 +0200 Subject: [PATCH 3/4] idf.py: ensure that build log is always sanitized from color sequences The actual output from the build tool (CMake/Ninja) may or may not contain color escape codes, depending on various factors. The output written to the log file should never include color escape codes, though. This is because color escape codes in files are usually not rendered as "color" in editors, and complicate reading. Also escape codes would break the regular expressions used to display hints for compilation errors. --- tools/idf_py_actions/tools.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index 854ca48a10..8822befae8 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -233,12 +233,6 @@ class RunTool: ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') return ansi_escape.sub('', text) - def prepare_for_print(out: str) -> str: - if not output_stream.isatty(): - # delete escape sequence if we printing in environments where ANSI coloring is disabled - return delete_ansi_escape(out) - return out - def print_progression(output: str) -> None: # Print a new line on top of the previous line sys.stdout.write('\x1b[K') @@ -275,8 +269,15 @@ class RunTool: output = await read_stream() if not output: break - output = prepare_for_print(output) - output_file.write(output) + output_noescape = delete_ansi_escape(output) + # Always remove escape sequences when writing the build log. + output_file.write(output_noescape) + # If idf.py output is redirected and the output stream is not a TTY, + # strip the escape sequences as well. + # (There shouldn't be any, but just in case.) + if not output_stream.isatty(): + output = output_noescape + if self.force_progression and output[0] == '[' and '-v' not in self.args and output_stream.isatty(): # print output in progression way but only the progression related (that started with '[') and if verbose flag is not set print_progression(output) From 1ba3d58b78a33af161fc3b97e72c44f6b6a07f5e Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Tue, 9 Aug 2022 02:37:59 +0200 Subject: [PATCH 4/4] tools: cmake: upgrade from 3.23.1 to 3.24.0 --- tools/tools.json | 50 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tools/tools.json b/tools/tools.json index a7936d3a3d..0594cb59a0 100644 --- a/tools/tools.json +++ b/tools/tools.json @@ -690,46 +690,46 @@ "versions": [ { "linux-amd64": { - "sha256": "f3c654b2e226b9d43369e0bd8487c51618d4dbe5a1af929dd32af7e6ca432d60", - "size": 45998644, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz" + "sha256": "726f88e6598523911e4bce9b059dc20b851aa77f97e4cc5573f4e42775a5c16f", + "size": 47042675, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-linux-x86_64.tar.gz" }, "linux-arm64": { - "sha256": "74062efddeb935bce3d33694a4db534cef9a650f77a9a153a9f217d9dc385c75", - "size": 47458032, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-aarch64.tar.gz" + "sha256": "50c3b8e9d3a3cde850dd1ea143df9d1ae546cbc5e74dc6d223eefc1979189651", + "size": 48478082, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-linux-aarch64.tar.gz" }, "linux-armel": { - "sha256": "aa6079237e16cc3b389479b2f7279d07e57f6aedad520e2b3014ef97fb906466", - "size": 19330381, - "url": "https://dl.espressif.com/dl/cmake/cmake-3.23.1-Linux-armv7l.tar.gz" + "sha256": "7dc787ef968dfef92491a4f191b8739ff70f8a649608b811c7a737b52481beb0", + "size": 19811327, + "url": "https://dl.espressif.com/dl/cmake/cmake-3.24.0-Linux-armv7l.tar.gz" }, "linux-armhf": { - "sha256": "aa6079237e16cc3b389479b2f7279d07e57f6aedad520e2b3014ef97fb906466", - "size": 19330381, - "url": "https://dl.espressif.com/dl/cmake/cmake-3.23.1-Linux-armv7l.tar.gz" + "sha256": "7dc787ef968dfef92491a4f191b8739ff70f8a649608b811c7a737b52481beb0", + "size": 19811327, + "url": "https://dl.espressif.com/dl/cmake/cmake-3.24.0-Linux-armv7l.tar.gz" }, "macos": { - "sha256": "f794ed92ccb4e9b6619a77328f313497d7decf8fb7e047ba35a348b838e0e1e2", - "size": 70988516, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-macos-universal.tar.gz" + "sha256": "3e0cca74a56d9027dabb845a5a26e42ef8e8b33beb1655d6a724187a345145e4", + "size": 72801419, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-macos-universal.tar.gz" }, "macos-arm64": { - "sha256": "f794ed92ccb4e9b6619a77328f313497d7decf8fb7e047ba35a348b838e0e1e2", - "size": 70988516, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-macos-universal.tar.gz" + "sha256": "3e0cca74a56d9027dabb845a5a26e42ef8e8b33beb1655d6a724187a345145e4", + "size": 72801419, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-macos-universal.tar.gz" }, - "name": "3.23.1", + "name": "3.24.0", "status": "recommended", "win32": { - "sha256": "9b509cc4eb7191dc128cfa3f2170036f9cbc7d9d5f93ff7fafc5b2d77b3b40dc", - "size": 39070972, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-windows-x86_64.zip" + "sha256": "b1ad8c2dbf0778e3efcc9fd61cd4a962e5c1af40aabdebee3d5074bcff2e103c", + "size": 40212531, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-windows-x86_64.zip" }, "win64": { - "sha256": "9b509cc4eb7191dc128cfa3f2170036f9cbc7d9d5f93ff7fafc5b2d77b3b40dc", - "size": 39070972, - "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-windows-x86_64.zip" + "sha256": "b1ad8c2dbf0778e3efcc9fd61cd4a962e5c1af40aabdebee3d5074bcff2e103c", + "size": 40212531, + "url": "https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-windows-x86_64.zip" } }, {