espressif/esp-idf
1
0
Fork 1
mirror of https://github.com/espressif/esp-idf.git synced 2025-10-03 10:30:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

change(ci): update regex to match soc headers in check_public_header.py

Browse Source
This commit is contained in:
Chen Jichang
2025-08-14 18:27:53 +08:00
committed by Gao Xu
parent 41f954facf
commit f041954ec3
1 changed files with 39 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
tools/ci/check_public_headers.py
View File

@@ -15,10 +15,6 @@ import subprocess
import tempfile import tempfile
from threading import Event from threading import Event
from threading import Thread from threading import Thread
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from typing import Union from typing import Union
@@ -38,7 +34,7 @@ class HeaderFailedBuildError(HeaderFailed):
self.compiler = compiler self.compiler = compiler
def __str__(self) -> str: def __str__(self) -> str:
return 'Header Build Error with {}'.format(self.compiler) return f'Header Build Error with {self.compiler}'
class HeaderFailedPreprocessError(HeaderFailed): class HeaderFailedPreprocessError(HeaderFailed):
@@ -63,17 +59,17 @@ class HeaderFailedContainsStaticAssert(HeaderFailed):
# Creates a temp file and returns both output as a string and a file name # Creates a temp file and returns both output as a string and a file name
# #
def exec_cmd_to_temp_file(what: List, suffix: str = '') -> Tuple[int, str, str, str, str]: def exec_cmd_to_temp_file(what: list, suffix: str = '') -> tuple[int, str, str, str, str]:
out_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False) out_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
rc, out, err, cmd = exec_cmd(what, out_file) rc, out, err, cmd = exec_cmd(what, out_file)
with open(out_file.name, 'r', encoding='utf-8') as f: with open(out_file.name, encoding='utf-8') as f:
out = f.read() out = f.read()
return rc, out, err, out_file.name, cmd return rc, out, err, out_file.name, cmd
def exec_cmd( def exec_cmd(
what: List, out_file: Union['tempfile._TemporaryFileWrapper[bytes]', int] = subprocess.PIPE what: list, out_file: Union['tempfile._TemporaryFileWrapper[bytes]', int] = subprocess.PIPE
) -> Tuple[int, str, str, str]: ) -> tuple[int, str, str, str]:
p = subprocess.Popen(what, stdin=subprocess.PIPE, stdout=out_file, stderr=subprocess.PIPE) p = subprocess.Popen(what, stdin=subprocess.PIPE, stdout=out_file, stderr=subprocess.PIPE)
output_b, err_b = p.communicate() output_b, err_b = p.communicate()
rc = p.returncode rc = p.returncode
@@ -87,9 +83,9 @@ class PublicHeaderChecker:
if self.verbose or debug: if self.verbose or debug:
print(message) print(message)
def __init__(self, verbose: bool = False, jobs: int = 1, prefix: Optional[str] = None) -> None: def __init__(self, verbose: bool = False, jobs: int = 1, prefix: str | None = None) -> None:
self.gcc = '{}gcc'.format(prefix) self.gcc = f'{prefix}gcc'
self.gpp = '{}g++'.format(prefix) self.gpp = f'{prefix}g++'
self.verbose = verbose self.verbose = verbose
self.jobs = jobs self.jobs = jobs
self.prefix = prefix self.prefix = prefix
@@ -100,10 +96,13 @@ class PublicHeaderChecker:
self.static_assert = re.compile(r'(_Static_assert|static_assert)') self.static_assert = re.compile(r'(_Static_assert|static_assert)')
self.defines_assert = re.compile(r'#define[ \t]+ESP_STATIC_ASSERT') self.defines_assert = re.compile(r'#define[ \t]+ESP_STATIC_ASSERT')
self.auto_soc_header = re.compile( self.auto_soc_header = re.compile(
r'components/soc/esp[a-z0-9_]+(?:/\w+)?/(include|register)/(soc|modem|hw_ver\d+/soc)/[a-zA-Z0-9_]+.h' r'components/soc/esp[a-z0-9_]+(?:/\w+)?/'
r'(include|register)/'
r'(soc|modem|hw_ver(?:\d+|_[A-Za-z0-9]+)/soc)/'
r'[a-zA-Z0-9_]+\.h$'
) )
self.assembly_nocode = r'^\s*(\.file|\.text|\.ident|\.option|\.attribute|(\.section)?).*$' self.assembly_nocode = r'^\s*(\.file|\.text|\.ident|\.option|\.attribute|(\.section)?).*$'
self.check_threads: List[Thread] = [] self.check_threads: list[Thread] = []
self.stdc = '--std=c99' self.stdc = '--std=c99'
self.stdcpp = '--std=c++17' self.stdcpp = '--std=c++17'
@@ -134,14 +133,14 @@ class PublicHeaderChecker:
try: try:
self.check_one_header(task, num) self.check_one_header(task, num)
except HeaderFailed as e: except HeaderFailed as e:
self.failed_queue.put('{}: Failed! {}'.format(task, e)) self.failed_queue.put(f'{task}: Failed! {e}')
except Exception as e: except Exception as e:
# Makes sure any unexpected exceptions causes the program to terminate # Makes sure any unexpected exceptions causes the program to terminate
self.failed_queue.put('{}: Failed! {}'.format(task, e)) self.failed_queue.put(f'{task}: Failed! {e}')
self.terminate.set() self.terminate.set()
raise raise
def get_failed(self) -> List: def get_failed(self) -> list:
return list(self.failed_queue.queue) return list(self.failed_queue.queue)
def join(self) -> None: def join(self) -> None:
@@ -166,9 +165,9 @@ class PublicHeaderChecker:
if not re.sub(self.assembly_nocode, '', out, flags=re.M).isspace(): if not re.sub(self.assembly_nocode, '', out, flags=re.M).isspace():
raise HeaderFailedContainsCode() raise HeaderFailedContainsCode()
return # Header OK: produced zero code return # Header OK: produced zero code
self.log('{}: FAILED: compilation issue'.format(header), True) self.log(f'{header}: FAILED: compilation issue', True)
self.log(err, True) self.log(err, True)
self.log('\nCompilation command failed:\n{}\n'.format(cmd), True) self.log(f'\nCompilation command failed:\n{cmd}\n', True)
raise HeaderFailedBuildError(compiler) raise HeaderFailedBuildError(compiler)
# Checks one header using preprocessing and parsing # Checks one header using preprocessing and parsing
@@ -214,7 +213,7 @@ class PublicHeaderChecker:
# Normalize the potential A//B, A/./B, A/../A, from the name # Normalize the potential A//B, A/./B, A/../A, from the name
normalized_path = os.path.normpath(header) normalized_path = os.path.normpath(header)
if grp and not re.search(self.defines_assert, out) and not re.search(self.auto_soc_header, normalized_path): if grp and not re.search(self.defines_assert, out) and not re.search(self.auto_soc_header, normalized_path):
self.log('{}: FAILED: contains {}. Please use ESP_STATIC_ASSERT'.format(header, grp.group(1)), True) self.log(f'{header}: FAILED: contains {grp.group(1)}. Please use ESP_STATIC_ASSERT', True)
raise HeaderFailedContainsStaticAssert() raise HeaderFailedContainsStaticAssert()
try: try:
# compile with C++, check for errors, outputs for a temp file # compile with C++, check for errors, outputs for a temp file
@@ -222,32 +221,32 @@ class PublicHeaderChecker:
if rc != 0: if rc != 0:
if re.search(self.error_macro, err): if re.search(self.error_macro, err):
if re.search(self.error_orphan_kconfig, err): if re.search(self.error_orphan_kconfig, err):
self.log('{}: CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED'.format(header), True) self.log(f'{header}: CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED', True)
raise HeaderFailedSdkconfig() raise HeaderFailedSdkconfig()
self.log('{}: Error directive failure: OK'.format(header)) self.log(f'{header}: Error directive failure: OK')
return return
self.log('{}: FAILED: compilation issue'.format(header), True) self.log(f'{header}: FAILED: compilation issue', True)
self.log(err, True) self.log(err, True)
self.log('\nCompilation command failed:\n{}\n'.format(cmd), True) self.log(f'\nCompilation command failed:\n{cmd}\n', True)
raise HeaderFailedPreprocessError() raise HeaderFailedPreprocessError()
# compile with C compiler, outputs to another temp file # compile with C compiler, outputs to another temp file
rc, _, err, c_out_file, _ = exec_cmd_to_temp_file([self.gcc, self.stdc] + all_compilation_flags) rc, _, err, c_out_file, _ = exec_cmd_to_temp_file([self.gcc, self.stdc] + all_compilation_flags)
if rc != 0: if rc != 0:
self.log('{} FAILED: compilation in C (while C++ compilation passes)'.format(header)) self.log(f'{header} FAILED: compilation in C (while C++ compilation passes)')
raise HeaderFailedPreprocessError() raise HeaderFailedPreprocessError()
# diff the two outputs # diff the two outputs
rc, diff, err, _ = exec_cmd(['diff', c_out_file, cpp_out_file]) rc, diff, err, _ = exec_cmd(['diff', c_out_file, cpp_out_file])
if not diff or diff.isspace(): if not diff or diff.isspace():
if not cpp_out or cpp_out.isspace(): if not cpp_out or cpp_out.isspace():
self.log('{} The same, but empty out - OK'.format(header)) self.log(f'{header} The same, but empty out - OK')
return return
self.log('{} FAILED C and C++ preprocessor output is the same!'.format(header), True) self.log(f'{header} FAILED C and C++ preprocessor output is the same!', True)
raise HeaderFailedCppGuardMissing() raise HeaderFailedCppGuardMissing()
if re.search(self.extern_c, diff): if re.search(self.extern_c, diff):
self.log('{} extern C present in the diff'.format(header)) self.log(f'{header} extern C present in the diff')
# now check the extern "C" is really in the unprocessed header # now check the extern "C" is really in the unprocessed header
if re.search(self.extern_c, out): if re.search(self.extern_c, out):
self.log('{} extern C present in the actual header, too - OK'.format(header)) self.log(f'{header} extern C present in the actual header, too - OK')
return return
# at this point we know that the header itself is missing extern-C, so we need to check if it # at this point we know that the header itself is missing extern-C, so we need to check if it
# contains an actual *code* we remove all preprocessor's directive to check if there's any code # contains an actual *code* we remove all preprocessor's directive to check if there's any code
@@ -255,19 +254,17 @@ class PublicHeaderChecker:
macros = re.compile(r'(?m)^\s*#(?:.*\\\r?\n)*.*$') # Matches multiline preprocessor directives macros = re.compile(r'(?m)^\s*#(?:.*\\\r?\n)*.*$') # Matches multiline preprocessor directives
without_macros = macros.sub('', out) without_macros = macros.sub('', out)
if without_macros.isspace(): if without_macros.isspace():
self.log("{} Header doesn't need extern-C, it's all just macros - OK".format(header)) self.log(f"{header} Header doesn't need extern-C, it's all just macros - OK")
return return
# at this point we know that the header is not only composed of macro definitions, but could # at this point we know that the header is not only composed of macro definitions, but could
# just contain some "harmless" macro calls let's remove them and check again # just contain some "harmless" macro calls let's remove them and check again
macros_calls = r'(.*?)ESP_STATIC_ASSERT[^;]+;' # static assert macro only, we could add more if needed macros_calls = r'(.*?)ESP_STATIC_ASSERT[^;]+;' # static assert macro only, we could add more if needed
without_macros = re.sub(macros_calls, '', without_macros, flags=re.DOTALL) without_macros = re.sub(macros_calls, '', without_macros, flags=re.DOTALL)
if without_macros.isspace(): if without_macros.isspace():
self.log( self.log(f"{header} Header doesn't need extern-C, it's all macros definitions and calls - OK")
"{} Header doesn't need extern-C, it's all macros definitions and calls - OK".format(header)
)
return return
self.log('{} Different but no extern C - FAILED'.format(header), True) self.log(f'{header} Different but no extern C - FAILED', True)
raise HeaderFailedCppGuardMissing() raise HeaderFailedCppGuardMissing()
finally: finally:
os.unlink(cpp_out_file) os.unlink(cpp_out_file)
@@ -277,9 +274,7 @@ class PublicHeaderChecker:
pass pass
# Get compilation data from an example to list all public header files # Get compilation data from an example to list all public header files
def list_public_headers( def list_public_headers(self, ignore_dirs: list, ignore_files: list | set, only_dir: str | None = None) -> None:
self, ignore_dirs: List, ignore_files: Union[List, Set], only_dir: Optional[str] = None
) -> None:
idf_path = os.getenv('IDF_PATH') idf_path = os.getenv('IDF_PATH')
if idf_path is None: if idf_path is None:
raise RuntimeError("Environment variable 'IDF_PATH' wasn't set.") raise RuntimeError("Environment variable 'IDF_PATH' wasn't set.")
@@ -294,13 +289,13 @@ class PublicHeaderChecker:
['idf.py', '-B', build_dir, f'-DSDKCONFIG={sdkconfig}', '-DCOMPONENTS=', 'reconfigure'], cwd=project_dir ['idf.py', '-B', build_dir, f'-DSDKCONFIG={sdkconfig}', '-DCOMPONENTS=', 'reconfigure'], cwd=project_dir
) )
def get_std(json: List, extension: str) -> str: def get_std(json: list, extension: str) -> str:
# compile commands for the files with specified extension, containing C(XX) standard flag # compile commands for the files with specified extension, containing C(XX) standard flag
command = [c for c in j if c['file'].endswith('.' + extension) and '-std=' in c['command']][0] command = [c for c in j if c['file'].endswith('.' + extension) and '-std=' in c['command']][0]
return str([s for s in command['command'].split() if 'std=' in s][0]) # grab the std flag return str([s for s in command['command'].split() if 'std=' in s][0]) # grab the std flag
build_commands_json = os.path.join(build_dir, 'compile_commands.json') build_commands_json = os.path.join(build_dir, 'compile_commands.json')
with open(build_commands_json, 'r', encoding='utf-8') as f: with open(build_commands_json, encoding='utf-8') as f:
j = json.load(f) j = json.load(f)
self.stdc = get_std(j, 'c') self.stdc = get_std(j, 'c')
self.stdcpp = get_std(j, 'cpp') self.stdcpp = get_std(j, 'cpp')
@@ -337,10 +332,10 @@ class PublicHeaderChecker:
if only_dir is not None and not os.path.relpath(d, idf_path).startswith( if only_dir is not None and not os.path.relpath(d, idf_path).startswith(
os.path.relpath(only_dir, idf_path) os.path.relpath(only_dir, idf_path)
): ):
self.log('{} - directory ignored (not in "{}")'.format(d, only_dir)) self.log(f'{d} - directory ignored (not in "{only_dir}")')
continue continue
if os.path.relpath(d, idf_path).startswith(tuple(ignore_dirs)): if os.path.relpath(d, idf_path).startswith(tuple(ignore_dirs)):
self.log('{} - directory ignored'.format(d)) self.log(f'{d} - directory ignored')
continue continue
for root, dirnames, filenames in os.walk(d): for root, dirnames, filenames in os.walk(d):
for filename in fnmatch.filter(filenames, '*.h'): for filename in fnmatch.filter(filenames, '*.h'):
@@ -352,10 +347,10 @@ class PublicHeaderChecker:
for file_name in all_include_files: for file_name in all_include_files:
rel_path_file = os.path.relpath(file_name, idf_path) rel_path_file = os.path.relpath(file_name, idf_path)
if any([os.path.commonprefix([d, rel_path_file]) == d for d in ignore_dirs]): if any([os.path.commonprefix([d, rel_path_file]) == d for d in ignore_dirs]):
self.log('{} - file ignored (inside ignore dir)'.format(file_name)) self.log(f'{file_name} - file ignored (inside ignore dir)')
continue continue
if rel_path_file in ignore_files: if rel_path_file in ignore_files:
self.log('{} - file ignored'.format(file_name)) self.log(f'{file_name} - file ignored')
continue continue
files_to_check.append(file_name) files_to_check.append(file_name)
# removes duplicates and places headers to a work queue # removes duplicates and places headers to a work queue
@@ -410,7 +405,7 @@ def check_all_headers() -> None:
# process excluded files and dirs # process excluded files and dirs
exclude_file = os.path.join(os.path.dirname(__file__), args.exclude_file) exclude_file = os.path.join(os.path.dirname(__file__), args.exclude_file)
with open(exclude_file, 'r', encoding='utf-8') as f: with open(exclude_file, encoding='utf-8') as f:
lines = [line.rstrip() for line in f] lines = [line.rstrip() for line in f]
ignore_files = [] ignore_files = []
ignore_dirs = [] ignore_dirs = []