diff --git a/tools/idf_py_actions/hint_modules/component_requirements.py b/tools/idf_py_actions/hint_modules/component_requirements.py new file mode 100644 index 0000000000..3ce6dd9dba --- /dev/null +++ b/tools/idf_py_actions/hint_modules/component_requirements.py @@ -0,0 +1,214 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import os +import re +from typing import Optional + +from idf_py_actions.tools import get_build_context + +''' +glossary: +orignal_component: component which compilation failed +source_component: component containing file which is including the missing header file +candidate_component: component which contain the missing header file +original_filename: abs path of file(compilation unit) in original_component +source_filename: abs path of file in source_component which is including the missing header file +missing_header: filename of the missing header included in source_filename +''' + +# Regex to find source_filename in preprocessor's error message +ENOENT_RE = re.compile(r'^(.+):\d+:\d+: fatal error: (.+): No such file or directory$', + flags=re.MULTILINE) +# Regex to find full preprocessor's error message to identify the original_filename +# in case the missing_header is reported in indirect include. +ENOENT_FULL_RE = re.compile(r'^(In file included.*No such file or directory)$', + flags=re.MULTILINE | re.DOTALL) +# Regex to find original_filename in preprocessor's error message +ORIGINAL_FILE_RE = re.compile(r'.*from (.*):[\d]+:') + + +def _get_absolute_path(filename: str, base: str) -> str: + # If filename path is relative, return absolute path based + # on base directory. The filename is normalized in any case. + if not os.path.isabs(filename): + filename = os.path.join(base, filename) + filename = os.path.normpath(filename) + return filename + + +def generate_hint(output: str) -> Optional[str]: + # get the project description + proj_desc = get_build_context().get('proj_desc') + if not proj_desc: + # hints cannot be generated because we are not in the build context, + # meaning ensure_build_directory() was not ran and project description + # is not available + return None + + hint_match = ENOENT_RE.search(output) + if not hint_match: + return None + + # this is the file where the error has occurred + source_filename = _get_absolute_path(hint_match.group(1), proj_desc['build_dir']) + + # this is the header file we tried to include + missing_header = hint_match.group(2) + + # find the source_component that contains the source file + found_source_component_name = None + found_source_component_info = None + for component_name, component_info in proj_desc['build_component_info'].items(): + # look if the source_filename is within a component directory, not only + # at component_info['sources'], because the missing file may be included + # from header file, which is not present in component_info['sources'] + component_dir = os.path.normpath(component_info['dir']) + if source_filename.startswith(component_dir): + found_source_component_name = component_name + found_source_component_info = component_info + break + if not found_source_component_name: + # The source file is not in any component. + # It could be in a subproject added via ExternalProject_Add, in which case + # we can't help much. + return None + + # find the original_component, which may be different from sourc_component + found_original_component_name = found_source_component_name + found_original_component_info = found_source_component_info + original_filename = source_filename + hint_match_full = ENOENT_FULL_RE.search(output) + if hint_match_full: + lines = hint_match_full.group().splitlines() + # second line from the end contains filename which is part of the + # original_component + original_file_match = ORIGINAL_FILE_RE.match(lines[-2]) + if original_file_match: + original_filename = _get_absolute_path(original_file_match.group(1), proj_desc['build_dir']) + for component_name, component_info in proj_desc['build_component_info'].items(): + component_dir = os.path.normpath(component_info['dir']) + if original_filename.startswith(component_dir): + found_original_component_name = component_name + found_original_component_info = component_info + break + else: + # We should never reach this path. It would probably mean + # the preprocessor output was changed. Anyway we can still + # report something meaningful, so just keep going. + pass + + # look for the header file in the public include directories of all components + found_dep_component_names = [] + for candidate_component_name, candidate_component_info in proj_desc['build_component_info'].items(): + if candidate_component_name == found_source_component_name: + # skip the component that contains the source file + continue + candidate_component_include_dirs = candidate_component_info['include_dirs'] + component_dir = os.path.normpath(candidate_component_info['dir']) + for candidate_component_include_dir in candidate_component_include_dirs: + candidate_header_path = os.path.join(component_dir, candidate_component_include_dir, missing_header) + if os.path.exists(candidate_header_path): + found_dep_component_names.append(candidate_component_name) + break # no need to look further in this component + + if not found_dep_component_names: + # Header file not found in any component INCLUDE_DIRS. Try to scan whole component + # directories if we can find the missing header there and notify user about possible + # missing entry in INCLUDE_DIRS. + candidate_component_include_dirs = [] + for component_name, component_info in proj_desc['build_component_info'].items(): + component_dir = os.path.normpath(component_info['dir']) + for root, _, _ in os.walk(component_dir): + full_path = os.path.normpath(os.path.join(root, missing_header)) + # sanity check that the full_path is still within component's directory + if not full_path.startswith(component_dir): + continue + if os.path.isfile(full_path): + candidate_component_include_dirs.append(f'{component_name}({full_path})') + + if candidate_component_include_dirs: + candidates = ', '.join(candidate_component_include_dirs) + return (f'Missing "{missing_header}" file name found in the following component(s): {candidates}. ' + f'Maybe one of the components needs to add the missing header directory to INCLUDE_DIRS ' + f'of idf_component_register call in CMakeLists.txt.') + + # The missing header not found anywhere, nothing much we can do here. + return None + + assert found_source_component_info is not None # to help mypy + assert found_original_component_info is not None # to help mypy + + # Sanity check: verify we didn't somehow find a component which is already in the requirements list + all_reqs = (found_source_component_info['reqs'] + + found_source_component_info['managed_reqs']) + if found_original_component_name == found_source_component_name: + # Add also private reqs, but only if source_component is same original_component. + # The missing_header may be part of component which is already added as private + # req for source_component. Meaning it's not part of source_component public + # interface. + all_reqs += (found_source_component_info['priv_reqs'] + + found_source_component_info['managed_priv_reqs']) + + for dep_component_name in found_dep_component_names: + if dep_component_name in all_reqs: + # Oops. This component is already in the requirements list. + # How did this happen? + return f'BUG: {missing_header} found in component {dep_component_name} which is already in the requirements list of {found_source_component_name}' + + # try to figure out the correct require type: REQUIRES or PRIV_REQUIRES + requires_type = None + source_component_has_priv_dep = False + if original_filename == source_filename: + # The error is reported directly in compilation unit, so + # missing_header should not be part of public interface. + requires_type = 'PRIV_REQUIRES' + elif found_original_component_name == found_source_component_name: + # The original_component and source_component are the same and original_filename + # is different from source_filename. Check if the source_file is part of the + # original_component's public interface. If so, the REQUIRES should be used. + for include_dir in found_original_component_info['include_dirs']: + include_dir = _get_absolute_path(found_original_component_info['dir'], include_dir) + if source_filename.startswith(include_dir): + # source_filename is part of public interface + requires_type = 'REQUIRES' + break + if not requires_type: + # source_file not part of public interface, suggest PRIV_REQUIRES + requires_type = 'PRIV_REQUIRES' + else: + # The source_filename is part of different component than the original_component, so + # the source_component needs to use REQUIRES to make the missing_header available for + # original_component. + requires_type = 'REQUIRES' + if len(found_dep_component_names) == 1: + # If there is only one component found as missing dependency, look at + # source_component private requires to see if the missing dependency is + # already there. If so, we suggest to move it from PRIV_REQUIRES to REQUIRES. + # This is done only if there is one component in found_dep_component_names, because + # otherwise we cannot be sure which component should be moved. + priv_reqs = (found_source_component_info['priv_reqs'] + + found_source_component_info['managed_priv_reqs']) + if found_dep_component_names[0] in priv_reqs: + source_component_has_priv_dep = True + + found_dep_component_names_list = ', '.join(found_dep_component_names) + source_filename_short = os.path.basename(source_filename) + cmakelists_file_to_fix = os.path.normpath(os.path.join(found_source_component_info['dir'], 'CMakeLists.txt')) + problem_description = ( + f'Compilation failed because {source_filename_short} (in "{found_source_component_name}" component) ' + f'includes {missing_header}, provided by {found_dep_component_names_list} component(s).\n') + + if source_component_has_priv_dep: + problem_solution = ( + f'However, {found_dep_component_names_list} component(s) is in the private requirements list ' + f'of "{found_source_component_name}".\n' + f'To fix this, move {found_dep_component_names_list} from PRIV_REQUIRES into ' + f'REQUIRES list of idf_component_register call in {cmakelists_file_to_fix}.') + else: + problem_solution = ( + f'However, {found_dep_component_names_list} component(s) is not in the requirements list ' + f'of "{found_source_component_name}".\n' + f'To fix this, add {found_dep_component_names_list} to {requires_type} list ' + f'of idf_component_register call in {cmakelists_file_to_fix}.') + + return problem_description + problem_solution