From 41706d8d548bf0e44edd3fcbc19de78684626dfd Mon Sep 17 00:00:00 2001 From: David Cermak Date: Tue, 21 Dec 2021 10:32:52 +0100 Subject: [PATCH 1/2] CI: Make public header check verbose on *sll* failures --- tools/ci/check_copyright_ignore.txt | 1 - tools/ci/check_public_headers.py | 17 +++-------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/tools/ci/check_copyright_ignore.txt b/tools/ci/check_copyright_ignore.txt index af366c6fdf..e8eb6406c0 100644 --- a/tools/ci/check_copyright_ignore.txt +++ b/tools/ci/check_copyright_ignore.txt @@ -2997,7 +2997,6 @@ tools/ci/check_build_warnings.py tools/ci/check_callgraph.py tools/ci/check_deprecated_kconfigs.py tools/ci/check_executables.py -tools/ci/check_public_headers.py tools/ci/check_readme_links.py tools/ci/check_rules_yml.py tools/ci/check_soc_struct_headers.py diff --git a/tools/ci/check_public_headers.py b/tools/ci/check_public_headers.py index 9b170d1b16..940998d511 100644 --- a/tools/ci/check_public_headers.py +++ b/tools/ci/check_public_headers.py @@ -2,19 +2,8 @@ # # Checks all public headers in IDF in the ci # -# Copyright 2020 Espressif Systems (Shanghai) PTE LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 # from __future__ import print_function, unicode_literals @@ -213,7 +202,7 @@ class PublicHeaderChecker: self.log('{}: Error directive failure: OK'.format(header)) return self.COMPILE_ERR_ERROR_MACRO_HDR_OK self.log('{}: FAILED: compilation issue'.format(header), True) - self.log(err) + self.log(err, True) return self.COMPILE_ERR_HDR_FAILED # compile with C compiler, outputs to another temp file rc, c99_out, err, c99_out_file = exec_cmd_to_temp_file([self.gcc, '--std=c99'] + all_compilation_flags) From 3ade0509c373de6f4163d0ce2188875ce12de526 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Tue, 21 Dec 2021 11:27:03 +0100 Subject: [PATCH 2/2] CI: Make public header checker print help/tips on failure Also prints failing compilation command to the log --- tools/ci/check_public_headers.py | 51 +++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/tools/ci/check_public_headers.py b/tools/ci/check_public_headers.py index 940998d511..1ff6a3410d 100644 --- a/tools/ci/check_public_headers.py +++ b/tools/ci/check_public_headers.py @@ -2,7 +2,7 @@ # # Checks all public headers in IDF in the ci # -# SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # @@ -49,10 +49,10 @@ class HeaderFailedContainsCode(HeaderFailed): # def exec_cmd_to_temp_file(what, suffix=''): out_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False) - rc, out, err = exec_cmd(what, out_file) + rc, out, err, cmd = exec_cmd(what, out_file) with open(out_file.name, 'r', encoding='utf-8') as f: out = f.read() - return rc, out, err, out_file.name + return rc, out, err, out_file.name, cmd def exec_cmd(what, out_file=subprocess.PIPE): @@ -61,7 +61,7 @@ def exec_cmd(what, out_file=subprocess.PIPE): rc = p.returncode output = output.decode('utf-8') if output is not None else None err = err.decode('utf-8') if err is not None else None - return rc, output, err + return rc, output, err, ' '.join(what) class PublicHeaderChecker: @@ -163,7 +163,7 @@ class PublicHeaderChecker: self.compile_one_header(header) temp_header = None try: - _, _, _, temp_header = exec_cmd_to_temp_file(['sed', '/#include/d; /#error/d', header], suffix='.h') + _, _, _, temp_header, _ = exec_cmd_to_temp_file(['sed', '/#include/d; /#error/d', header], suffix='.h') res = self.preprocess_one_header(temp_header, num, ignore_sdkconfig_issue=True) if res == self.PREPROC_OUT_SAME_HRD_FAILED: raise HeaderFailedCppGuardMissing() @@ -174,26 +174,27 @@ class PublicHeaderChecker: os.unlink(temp_header) def compile_one_header(self, header): - rc, out, err = exec_cmd([self.gcc, '-S', '-o-', '-include', header, self.main_c] + self.include_dir_flags) + rc, out, err, cmd = exec_cmd([self.gcc, '-S', '-o-', '-include', header, self.main_c] + self.include_dir_flags) if rc == 0: if not re.sub(self.assembly_nocode, '', out, flags=re.M).isspace(): raise HeaderFailedContainsCode() return # Header OK: produced zero code self.log('{}: FAILED: compilation issue'.format(header), True) self.log(err, True) + self.log('\nCompilation command failed:\n{}\n'.format(cmd), True) raise HeaderFailedBuildError() def preprocess_one_header(self, header, num, ignore_sdkconfig_issue=False): all_compilation_flags = ['-w', '-P', '-E', '-DESP_PLATFORM', '-include', header, self.main_c] + self.include_dir_flags if not ignore_sdkconfig_issue: # just strip commnets to check for CONFIG_... macros - rc, out, err = exec_cmd([self.gcc, '-fpreprocessed', '-dD', '-P', '-E', header] + self.include_dir_flags) + rc, out, err, _ = exec_cmd([self.gcc, '-fpreprocessed', '-dD', '-P', '-E', header] + self.include_dir_flags) if re.search(self.kconfig_macro, out): # enable defined #error if sdkconfig.h not included all_compilation_flags.append('-DIDF_CHECK_SDKCONFIG_INCLUDED') try: # compile with C++, check for errors, outputs for a temp file - rc, cpp_out, err, cpp_out_file = exec_cmd_to_temp_file([self.gpp, '--std=c++17'] + all_compilation_flags) + rc, cpp_out, err, cpp_out_file, cmd = exec_cmd_to_temp_file([self.gpp, '--std=c++17'] + all_compilation_flags) if rc != 0: if re.search(self.error_macro, err): if re.search(self.error_orphan_kconfig, err): @@ -203,14 +204,15 @@ class PublicHeaderChecker: return self.COMPILE_ERR_ERROR_MACRO_HDR_OK self.log('{}: FAILED: compilation issue'.format(header), True) self.log(err, True) + self.log('\nCompilation command failed:\n{}\n'.format(cmd), True) return self.COMPILE_ERR_HDR_FAILED # compile with C compiler, outputs to another temp file - rc, c99_out, err, c99_out_file = exec_cmd_to_temp_file([self.gcc, '--std=c99'] + all_compilation_flags) + rc, _, err, c99_out_file, _ = exec_cmd_to_temp_file([self.gcc, '--std=c99'] + all_compilation_flags) if rc != 0: self.log('{} FAILED should never happen'.format(header)) return self.COMPILE_ERR_HDR_FAILED # diff the two outputs - rc, diff, err = exec_cmd(['diff', c99_out_file, cpp_out_file]) + rc, diff, err, _ = exec_cmd(['diff', c99_out_file, cpp_out_file]) if not diff or diff.isspace(): if not cpp_out or cpp_out.isspace(): self.log('{} The same, but empty out - OK'.format(header)) @@ -291,7 +293,33 @@ class PublicHeaderChecker: def check_all_headers(): - parser = argparse.ArgumentParser('Public header checker file') + parser = argparse.ArgumentParser('Public header checker file', formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''\ + Tips for fixing failures reported by this script + ------------------------------------------------ + This checker validates all public headers to detect these types of issues: + 1) "Sdkconfig Error": Using SDK config macros without including "sdkconfig.h" + * Check if the failing include file or any other included file uses "CONFIG_..." prefixed macros + 2) "Header Build Error": Header itself is not compilable (missing includes, macros, types) + * Check that all referenced macros, types are available (defined or included) + * Check that all included header files are available (included in paths) + * Check for possible compilation issues + * Try to compile only the offending header file + 3) "Header Missing C++ Guard": Preprocessing the header by C and C++ should produce different output + * Check if the "#ifdef __cplusplus" header sentinels are present + 4) "Header Produced non-zero object": Header contains some object, a definition + * Check if no definition is present in the offending header file + + Notes: + * The script validates *all* header files (recursively) in public folders for all components. + * The script locates include paths from running a default build of "examples/get-started/blink' + * The script does not support any other targets than esp32 + + General tips: + * Use "-d" argument to make the script check only the offending header file + * Use "-v" argument to produce more verbose output + * Copy, paste and execute the compilation commands to reproduce build errors (script prints out + the entire compilation command line with absolute paths) + ''') parser.add_argument('--verbose', '-v', help='enables verbose mode', action='store_true') parser.add_argument('--jobs', '-j', help='number of jobs to run checker', default=1, type=int) parser.add_argument('--prefix', '-p', help='compiler prefix', default='xtensa-esp32-elf-', type=str) @@ -322,6 +350,7 @@ def check_all_headers(): if len(failures) > 0: for failed in failures: print(failed) + print(parser.epilog) exit(1) print('No errors found') except KeyboardInterrupt: