From 397fdb56fb80807a21e2800bc44f356a0ab6de07 Mon Sep 17 00:00:00 2001 From: Aleksei Apaseev Date: Wed, 18 Sep 2024 11:16:04 +0800 Subject: [PATCH] ci: ensure binary size artifacts are uploaded in Build Stage Child pipeline via artifacts_handler script. - Updated dynamic pipeline to include artifact upload commands for size reports, allowing easier parsing on the CI dashboard. - Each job now handles its own size.json file, ensuring independent access for better CI artifact management. --- .../dynamic_pipelines/scripts/child_pipeline_build_apps.py | 2 +- tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml | 1 + tools/ci/idf_ci/uploader.py | 7 +++---- tools/ci/idf_pytest/constants.py | 1 + tools/ci/idf_pytest/script.py | 3 ++- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py b/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py index 1f31deb539..a76631f702 100644 --- a/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py +++ b/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py @@ -66,7 +66,7 @@ if __name__ == '__main__': check_app_dependencies=True, parallel_count=args.parallel_count, parallel_index=args.parallel_index, - collect_size_info='size_info_@p.txt', + collect_size_info='size_info.txt', collect_app_info=args.collect_app_info, junitxml=args.junitxml, copy_sdkconfig=True, diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 4316648672..3eff163539 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -37,6 +37,7 @@ --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" --modified-components ${MR_MODIFIED_COMPONENTS} --junitxml "build_summary_${CI_JOB_NAME_SLUG}.xml" + - run_cmd python tools/ci/artifacts_handler.py upload --type size_reports .dynamic_target_test_template: extends: diff --git a/tools/ci/idf_ci/uploader.py b/tools/ci/idf_ci/uploader.py index 8310c3e11a..8ecff4ccbc 100644 --- a/tools/ci/idf_ci/uploader.py +++ b/tools/ci/idf_ci/uploader.py @@ -16,6 +16,7 @@ from idf_build_apps import App from idf_build_apps.utils import rmdir from idf_ci_utils import IDF_PATH from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME +from idf_pytest.constants import DEFAULT_SIZE_JSON_FILENAME class AppDownloader: @@ -60,9 +61,6 @@ class AppUploader(AppDownloader): ArtifactType.LOGS: [ DEFAULT_BUILD_LOG_FILENAME, ], - ArtifactType.SIZE_REPORTS: [ - 'size.json', - ], } def __init__(self, pipeline_id: t.Union[str, int, None] = None) -> None: @@ -108,12 +106,13 @@ class AppUploader(AppDownloader): else: upload_types = [artifact_type] + # Upload of size.json files is handled by GitLab CI via "artifacts_handler.py" script. print(f'Uploading {app_build_path} {[k.value for k in upload_types]} to minio server') for upload_type in upload_types: uploaded |= self._upload_app(app_build_path, upload_type) if uploaded: - rmdir(app_build_path, exclude_file_patterns=DEFAULT_BUILD_LOG_FILENAME) + rmdir(app_build_path, exclude_file_patterns=[DEFAULT_BUILD_LOG_FILENAME, DEFAULT_SIZE_JSON_FILENAME]) def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None: app_path, build_dir = os.path.split(app_build_path) diff --git a/tools/ci/idf_pytest/constants.py b/tools/ci/idf_pytest/constants.py index 5039f615f1..dadb57fd8d 100644 --- a/tools/ci/idf_pytest/constants.py +++ b/tools/ci/idf_pytest/constants.py @@ -153,6 +153,7 @@ DEFAULT_FULL_BUILD_TEST_FILEPATTERNS = [ 'tools/ci/ignore_build_warnings.txt', ] DEFAULT_BUILD_LOG_FILENAME = 'build_log.txt' +DEFAULT_SIZE_JSON_FILENAME = 'size.json' class CollectMode(str, Enum): diff --git a/tools/ci/idf_pytest/script.py b/tools/ci/idf_pytest/script.py index be99053ef6..0ff8b5e14f 100644 --- a/tools/ci/idf_pytest/script.py +++ b/tools/ci/idf_pytest/script.py @@ -25,6 +25,7 @@ from idf_py_actions.constants import SUPPORTED_TARGETS as TOOLS_SUPPORTED_TARGET from .constants import CollectMode from .constants import DEFAULT_BUILD_LOG_FILENAME from .constants import DEFAULT_CONFIG_RULES_STR +from .constants import DEFAULT_SIZE_JSON_FILENAME from .constants import PytestCase from .plugin import IdfPytestEmbedded @@ -162,7 +163,7 @@ def get_all_apps( build_dir='build_@t_@w', config_rules_str=config_rules_str or DEFAULT_CONFIG_RULES_STR, build_log_filename=DEFAULT_BUILD_LOG_FILENAME, - size_json_filename='size.json', + size_json_filename=DEFAULT_SIZE_JSON_FILENAME, check_warnings=True, manifest_rootpath=IDF_PATH, compare_manifest_sha_filepath=compare_manifest_sha_filepath,