diff --git a/.gitignore b/.gitignore index f70980b703..313692055c 100644 --- a/.gitignore +++ b/.gitignore @@ -102,6 +102,7 @@ pytest_embedded_log/ list_job*.txt size_info*.txt XUNIT_RESULT*.xml +.manifest_sha # clang config (for LSP) .clangd diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 4e126a0919..b757d53666 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -265,6 +265,8 @@ generate_build_child_pipeline: dependencies: # set dependencies to null to avoid missing artifacts issue needs: - pipeline_variables + - job: baseline_manifest_sha + optional: true artifacts: paths: - build_child_pipeline.yml diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 5e139646a8..80b24b9ee5 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -169,3 +169,35 @@ pipeline_variables: - pipeline.env expire_in: 1 week when: always + +baseline_manifest_sha: + extends: + - .pre_check_template + - .rules:dev-push + tags: [fast_run, shiny] + script: + - | + # merged results pipelines, by default + # diff between target-branch-head and merged-result-head + if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then + git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1 + git checkout FETCH_HEAD + idf-build-apps dump-manifest-sha \ + --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ + --output .manifest_sha + # merge request pipelines, when the mr got conflicts + # diff between diff-base-sha and merge-request-head + elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then + git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 + git checkout FETCH_HEAD + idf-build-apps dump-manifest-sha \ + --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ + --output .manifest_sha + # other pipelines, like the protected branches pipelines + # not triggered in this job + fi + artifacts: + paths: + - .manifest_sha + expire_in: 1 week + when: always diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index a9557cf916..b68f6e6056 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -217,6 +217,10 @@ rules: - <<: *if-tag-release +.rules:dev-push: + rules: + - <<: *if-dev-push + # Do not upload caches on dev branches by default .rules:upload-python-cache: rules: diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py index 7c8bf1d4d1..5941ce4dcd 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py @@ -89,6 +89,7 @@ def main(arguments: argparse.Namespace) -> None: marker_expr='not host_test', config_rules_str=DEFAULT_CONFIG_RULES_STR, extra_default_build_targets=extra_default_build_targets, + compare_manifest_sha_filepath=arguments.compare_manifest_sha_filepath, modified_components=arguments.modified_components, modified_files=arguments.modified_files, ignore_app_dependencies_filepatterns=arguments.ignore_app_dependencies_filepatterns, @@ -165,6 +166,11 @@ if __name__ == '__main__': default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), help='default build test rules config file', ) + parser.add_argument( + '--compare-manifest-sha-filepath', + default=os.path.join(IDF_PATH, '.manifest_sha'), + help='Path to the recorded manifest sha file generated by `idf-build-apps dump-manifest-sha`' + ) parser.add_argument( '--modified-components', type=_separate_str_to_list, @@ -237,4 +243,8 @@ if __name__ == '__main__': # setting default values args.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS + if not os.path.isfile(args.compare_manifest_sha_filepath): + # ignore if the file does not exist + args.compare_manifest_sha_filepath = None + main(args) diff --git a/tools/ci/idf_pytest/script.py b/tools/ci/idf_pytest/script.py index eed0609e0f..0a2770bf5e 100644 --- a/tools/ci/idf_pytest/script.py +++ b/tools/ci/idf_pytest/script.py @@ -128,6 +128,7 @@ def get_all_apps( config_rules_str: t.Optional[t.List[str]] = None, preserve_all: bool = False, extra_default_build_targets: t.Optional[t.List[str]] = None, + compare_manifest_sha_filepath: t.Optional[str] = None, modified_components: t.Optional[t.List[str]] = None, modified_files: t.Optional[t.List[str]] = None, ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, @@ -142,8 +143,10 @@ def get_all_apps( :param config_rules_str: config rules string :param preserve_all: preserve all apps :param extra_default_build_targets: extra default build targets + :param compare_manifest_sha_filepath: check manifest sha filepath :param modified_components: modified components :param modified_files: modified files + :param ignore_app_dependencies_components: ignore app dependencies components :param ignore_app_dependencies_filepatterns: ignore app dependencies filepatterns :return: tuple of test-required apps and non-test-related apps """ @@ -161,6 +164,7 @@ def get_all_apps( size_json_filename='size.json', check_warnings=True, manifest_rootpath=IDF_PATH, + compare_manifest_sha_filepath=compare_manifest_sha_filepath, manifest_files=get_all_manifest_files(), default_build_targets=SUPPORTED_TARGETS + (extra_default_build_targets or []), modified_components=modified_components,