From 270e7dec384c04da940f537e322889ab38a6bc73 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Sun, 28 Jun 2020 13:56:04 +0800 Subject: [PATCH] update after MR reviews --- tools/build_apps.py | 9 +- tools/ci/build_examples.sh | 8 +- tools/ci/build_test_apps.sh | 7 +- tools/ci/build_unit_test.sh | 3 +- tools/ci/check_build_warnings.py | 6 +- tools/ci/config/build.yml | 3 +- tools/ci/config/pre_check.yml | 5 +- .../python_packages/ttfw_idf/CIScanTests.py | 30 +++++-- tools/find_apps.py | 83 ++++++++++++------- 9 files changed, 92 insertions(+), 62 deletions(-) diff --git a/tools/build_apps.py b/tools/build_apps.py index be83347f95..9d2969ca3f 100755 --- a/tools/build_apps.py +++ b/tools/build_apps.py @@ -5,9 +5,10 @@ # import argparse +import logging import shutil import sys -import logging + from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS @@ -109,7 +110,7 @@ def main(): failed_builds = [] for build_info in builds_for_current_job: if not build_info.build: - logging.info('Skip build detected. Skipping...') + logging.info("Skip building app {}".format(build_info.app_dir)) continue logging.info("Running build {}: {}".format(build_info.index, repr(build_info))) @@ -124,9 +125,9 @@ def main(): raise SystemExit(1) else: if not build_info.preserve: - logging.info('NOT preserve artifacts detected. Deleting...') + logging.info("Removing build directory {}".format(build_info.build_dir)) # we only remove binaries here, log files are still needed by check_build_warnings.py - shutil.rmtree(build_info.work_dir, ignore_errors=True) + shutil.rmtree(build_info.build_dir, ignore_errors=True) if failed_builds: logging.error("The following build have failed:") diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh index 4a5915d101..df7e4745a3 100755 --- a/tools/ci/build_examples.sh +++ b/tools/ci/build_examples.sh @@ -72,13 +72,9 @@ cd ${IDF_PATH} # If changing the work-dir or build-dir format, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py. -${IDF_PATH}/tools/find_apps.py examples \ +${IDF_PATH}/tools/find_apps.py \ -vv \ --format json \ - --build-system ${EXAMPLE_TEST_BUILD_SYSTEM} \ - --target ${IDF_TARGET} \ - --recursive \ - --exclude examples/build_system/idf_as_lib \ --work-dir "${BUILD_PATH}/@f/@w/@t" \ --build-dir build \ --build-log "${LOG_PATH}/@f_@w.txt" \ @@ -86,7 +82,7 @@ ${IDF_PATH}/tools/find_apps.py examples \ --config 'sdkconfig.ci=default' \ --config 'sdkconfig.ci.*=' \ --config '=default' \ - --scan-tests-json ${SCAN_EXAMPLE_TEST_JSON} + --app-list ${SCAN_EXAMPLE_TEST_JSON} # --config rules above explained: # 1. If sdkconfig.ci exists, use it build the example with configuration name "default" diff --git a/tools/ci/build_test_apps.sh b/tools/ci/build_test_apps.sh index 627d1bb781..7909044f07 100755 --- a/tools/ci/build_test_apps.sh +++ b/tools/ci/build_test_apps.sh @@ -62,12 +62,9 @@ cd ${IDF_PATH} # If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py. -${IDF_PATH}/tools/find_apps.py tools/test_apps \ +${IDF_PATH}/tools/find_apps.py \ -vv \ --format json \ - --build-system cmake \ - --target ${IDF_TARGET} \ - --recursive \ --work-dir "${BUILD_PATH}/@f/@w/@t" \ --build-dir build \ --build-log "${LOG_PATH}/@f_@w.txt" \ @@ -75,7 +72,7 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \ --config 'sdkconfig.ci=default' \ --config 'sdkconfig.ci.*=' \ --config '=default' \ - --scan-tests-json ${SCAN_CUSTOM_TEST_JSON} + --app-list ${SCAN_CUSTOM_TEST_JSON} # --config rules above explained: # 1. If sdkconfig.ci exists, use it build the example with configuration name "default" diff --git a/tools/ci/build_unit_test.sh b/tools/ci/build_unit_test.sh index dd4402f1d9..deeb0e8cd3 100755 --- a/tools/ci/build_unit_test.sh +++ b/tools/ci/build_unit_test.sh @@ -63,7 +63,8 @@ cd ${IDF_PATH} # This part of the script produces the same result for all the unit test app build jobs. It may be moved to a separate stage # (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact. -${IDF_PATH}/tools/find_apps.py tools/unit-test-app \ +${IDF_PATH}/tools/find_apps.py \ + -p tools/unit-test-app \ -vv \ --format json \ --build-system cmake \ diff --git a/tools/ci/check_build_warnings.py b/tools/ci/check_build_warnings.py index bb01035ac5..0d63869a31 100755 --- a/tools/ci/check_build_warnings.py +++ b/tools/ci/check_build_warnings.py @@ -6,11 +6,11 @@ # log files for every build. # Exits with a non-zero exit code if any warning is found. -import os -import sys import argparse import logging +import os import re +import sys try: from find_build_apps import BuildItem, setup_logging @@ -78,7 +78,7 @@ def main(): found_warnings = 0 for build_item in build_items: if not build_item.build: - logging.debug('Skipping build detected. Skipping checking...') + logging.debug("Skip checking build log for app {}".format(build_item.app_dir)) continue if not build_item.build_log_path: logging.debug("No log file for {}".format(build_item.work_dir)) diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 3d56725bfb..d09977aef5 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -176,7 +176,8 @@ build_examples_cmake_esp32s2: variables: LOG_PATH: "${CI_PROJECT_DIR}/log_test_apps" BUILD_PATH: "${CI_PROJECT_DIR}/build_test_apps" - SCAN_CUSTOM_TEST_JSON: ${CI_PROJECT_DIR}/tools/test_apps/test_configs/scan_${IDF_TARGET}.json + CUSTOM_TEST_BUILD_SYSTEM: "cmake" + SCAN_CUSTOM_TEST_JSON: ${CI_PROJECT_DIR}/tools/test_apps/test_configs/scan_${IDF_TARGET}_${CUSTOM_TEST_BUILD_SYSTEM}.json only: variables: - $BOT_TRIGGER_WITH_LABEL == null diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml index 7d2b80ec61..a300f44139 100644 --- a/tools/ci/config/pre_check.yml +++ b/tools/ci/config/pre_check.yml @@ -203,6 +203,7 @@ scan_tests: extends: .scan_build_tests only: variables: + - $BOT_TRIGGER_WITH_LABEL == null - $BOT_LABEL_EXAMPLE_TEST - $BOT_LABEL_CUSTOM_TEST artifacts: @@ -215,6 +216,6 @@ scan_tests: TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs script: - - python $CI_SCAN_TESTS_PY example_test -b make $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR - - python $CI_SCAN_TESTS_PY example_test -b cmake $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR + - python $CI_SCAN_TESTS_PY example_test -b make $EXAMPLE_TEST_DIR --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR + - python $CI_SCAN_TESTS_PY example_test -b cmake $EXAMPLE_TEST_DIR --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR - python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py index fbac2bb9c7..88764d4f3b 100644 --- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -52,13 +52,21 @@ def main(): actions = parser.add_subparsers(dest='action') common = argparse.ArgumentParser(add_help=False) - common.add_argument('paths', type=str, nargs='+', + common.add_argument('paths', + nargs='+', help="One or more app paths") - common.add_argument('-b', '--build-system', choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE) - common.add_argument('-c', '--ci-config-file', type=str, required=True, + common.add_argument('-b', '--build-system', + choices=BUILD_SYSTEMS.keys(), + default=BUILD_SYSTEM_CMAKE) + common.add_argument('-c', '--ci-config-file', + required=True, help="gitlab ci config target-test file") - common.add_argument('-o', '--output-path', type=str, required=True, + common.add_argument('-o', '--output-path', + required=True, help="output path of the scan result") + common.add_argument("--exclude", + action="append", + help="Ignore specified directory. Can be used multiple times.") common.add_argument('--preserve', action="store_true", help='add this flag to preserve artifacts for all apps') common.add_argument('--build-all', action="store_true", @@ -99,7 +107,8 @@ def main(): ''' scan_info_dict = defaultdict(dict) # store the test cases dir, exclude these folders when scan for standalone apps - exclude_apps = [] + default_exclude = args.exclude if args.exclude else [] + exclude_apps = default_exclude build_system = args.build_system.lower() build_system_class = BUILD_SYSTEMS[build_system] @@ -112,7 +121,7 @@ def main(): app_target = case.case_info['target'] if app_target.lower() != target.lower(): continue - test_case_apps.update(find_apps(build_system_class, app_dir, True, [], target.lower())) + test_case_apps.update(find_apps(build_system_class, app_dir, True, default_exclude, target.lower())) exclude_apps.append(app_dir) for target in VALID_TARGETS: @@ -130,18 +139,21 @@ def main(): apps.append({ 'app_dir': app_dir, 'build': True, + 'build_system': args.build_system, + 'target': target, 'preserve': args.preserve or test_case_apps_preserve_default }) for app_dir in scan_info_dict[target]['standalone_apps']: apps.append({ 'app_dir': app_dir, 'build': build_all if build_system == 'cmake' else True, + 'build_system': args.build_system, + 'target': target, 'preserve': (args.preserve and build_all) if build_system == 'cmake' else False }) output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system)) - if apps: - with open(output_path, 'w') as fw: - fw.writelines([json.dumps(app) + '\n' for app in apps]) + with open(output_path, 'w') as fw: + fw.writelines([json.dumps(app) + '\n' for app in apps]) if __name__ == '__main__': diff --git a/tools/find_apps.py b/tools/find_apps.py index a7cfe0872f..0fe856b484 100755 --- a/tools/find_apps.py +++ b/tools/find_apps.py @@ -5,12 +5,13 @@ # Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds. import argparse -import json -import os -import sys -import re import glob +import json import logging +import os +import re +import sys + import typing from find_build_apps import ( @@ -197,7 +198,10 @@ def main(): action="store_true", help="Look for apps in the specified directories recursively.", ) - parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE) + parser.add_argument( + "--build-system", + choices=BUILD_SYSTEMS.keys() + ) parser.add_argument( "--work-dir", help="If set, the app is first copied into the specified directory, and then built." + @@ -241,60 +245,77 @@ def main(): help="Output the list of builds to the specified file", ) parser.add_argument( - '-s', - '--scan-tests-json', + "--app-list", default=None, - help="Scan tests result. Restrict the build/architect behavior to apps need to be built.\n" + help="Scan tests results. Restrict the build/artifacts preservation behavior to apps need to be built. " "If the file does not exist, will build all apps and upload all artifacts." ) - parser.add_argument("paths", nargs="+", help="One or more app paths.") + parser.add_argument( + "-p", "--paths", + nargs="+", + help="One or more app paths." + ) args = parser.parse_args() setup_logging(args) - build_system_class = BUILD_SYSTEMS[args.build_system] - - # If the build target is not set explicitly, get it from the environment or use the default one (esp32) - if not args.target: - env_target = os.environ.get("IDF_TARGET") - if env_target: - logging.info("--target argument not set, using IDF_TARGET={} from the environment".format(env_target)) - args.target = env_target - else: - logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET)) - args.target = DEFAULT_TARGET + # Arguments Validation + if args.app_list: + conflict_args = [args.recursive, args.build_system, args.target, args.exclude, args.paths] + if any(conflict_args): + raise ValueError('Conflict settings. "recursive", "build_system", "target", "exclude", "paths" should not ' + 'be specified with "app_list"') + if not os.path.exists(args.app_list): + raise OSError("File not found {}".format(args.app_list)) + else: + # If the build target is not set explicitly, get it from the environment or use the default one (esp32) + if not args.target: + env_target = os.environ.get("IDF_TARGET") + if env_target: + logging.info("--target argument not set, using IDF_TARGET={} from the environment".format(env_target)) + args.target = env_target + else: + logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET)) + args.target = DEFAULT_TARGET + if not args.build_system: + logging.info("--build-system argument not set, using {} as the default".format(BUILD_SYSTEM_CMAKE)) + args.build_system = BUILD_SYSTEM_CMAKE + required_args = [args.build_system, args.target, args.paths] + if not all(required_args): + raise ValueError('If app_list not set, arguments "build_system", "target", "paths" are required.') # Prepare the list of app paths, try to read from the scan_tests result. - # If the file exists, then follow the file's app_dir and build/archifacts behavior, won't do find_apps() again. + # If the file exists, then follow the file's app_dir and build/artifacts behavior, won't do find_apps() again. # If the file not exists, will do find_apps() first, then build all apps and upload all artifacts. - if args.scan_tests_json and os.path.exists(args.scan_tests_json): - apps = [json.loads(line) for line in open(args.scan_tests_json)] + if args.app_list: + apps = [json.loads(line) for line in open(args.app_list)] else: app_dirs = [] + build_system_class = BUILD_SYSTEMS[args.build_system] for path in args.paths: app_dirs += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target) - apps = [{'app_dir': app_dir, 'build': True, 'preserve': True} for app_dir in app_dirs] + apps = [{"app_dir": app_dir, "build": True, "preserve": True} for app_dir in app_dirs] if not apps: - logging.critical("No {} apps found".format(build_system_class.NAME)) + logging.critical("No apps found") raise SystemExit(1) logging.info("Found {} apps".format(len(apps))) - apps.sort(key=lambda x: x['app_dir']) + apps.sort(key=lambda x: x["app_dir"]) # Find compatible configurations of each app, collect them as BuildItems build_items = [] # type: typing.List[BuildItem] config_rules = config_rules_from_str(args.config or []) for app in apps: build_items += find_builds_for_app( - app['app_dir'], + app["app_dir"], args.work_dir, args.build_dir, args.build_log, - args.target, - args.build_system, + args.target or app["target"], + args.build_system or app["build_system"], config_rules, - app['build'], - app['preserve'], + app["build"], + app["preserve"], ) logging.info("Found {} builds".format(len(build_items)))