Merge branch 'feature/scan_test_at_pre_check' into 'master'

ci: scan_tests at pre_check stage to determine build/artifact behavior for example_test and custom_test

Closes IDF-1376

See merge request espressif/esp-idf!8447

examples/peripherals/temp_sensor_esp32s2/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32-S2 |
+| ----------------- | -------- |
+
 # ESP32-S2 Temperature Sensor Example
 
 The ESP32-S2 has a built-in temperature sensor. The temperature sensor module contains an 8-bit Sigma-Delta ADC and a temperature offset DAC.    

tools/build_apps.py

@@ -5,8 +5,10 @@
 #
 
 import argparse
-import sys
 import logging
+import shutil
+import sys
+
 from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS
 
 
@@ -75,10 +77,9 @@ def main():
     setup_logging(args)
 
     build_items = [BuildItem.from_json(line) for line in args.build_list]
-
     if not build_items:
-        logging.error("Empty build list!")
+        logging.warning("Empty build list")
-        raise SystemExit(1)
+        SystemExit(0)
 
     num_builds = len(build_items)
     num_jobs = args.parallel_count
@@ -117,6 +118,11 @@ def main():
                 failed_builds.append(build_info)
             else:
                 raise SystemExit(1)
+        else:
+            if not build_info.preserve:
+                logging.info("Removing build directory {}".format(build_info.build_dir))
+                # we only remove binaries here, log files are still needed by check_build_warnings.py
+                shutil.rmtree(build_info.build_dir, ignore_errors=True)
 
     if failed_builds:
         logging.error("The following build have failed:")

tools/ci/build_examples.sh

@@ -31,6 +31,7 @@ die() {
 [ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set"
 [ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set"
 [ -z ${EXAMPLE_TEST_BUILD_SYSTEM} ] && die "EXAMPLE_TEST_BUILD_SYSTEM is not set"
+[ -z ${SCAN_EXAMPLE_TEST_JSON} ] && die "SCAN_EXAMPLE_TEST_JSON is not set"
 [ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
 [ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH}
 
@@ -71,13 +72,9 @@ cd ${IDF_PATH}
 
 # If changing the work-dir or build-dir format, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
 
-${IDF_PATH}/tools/find_apps.py examples \
+${IDF_PATH}/tools/find_apps.py \
     -vv \
     --format json \
-    --build-system ${EXAMPLE_TEST_BUILD_SYSTEM} \
-    --target ${IDF_TARGET} \
-    --recursive \
-    --exclude examples/build_system/idf_as_lib \
     --work-dir "${BUILD_PATH}/@f/@w/@t" \
     --build-dir build \
     --build-log "${LOG_PATH}/@f_@w.txt" \
@@ -85,6 +82,7 @@ ${IDF_PATH}/tools/find_apps.py examples \
     --config 'sdkconfig.ci=default' \
     --config 'sdkconfig.ci.*=' \
     --config '=default' \
+    --app-list ${SCAN_EXAMPLE_TEST_JSON}
 
 # --config rules above explained:
 # 1. If sdkconfig.ci exists, use it build the example with configuration name "default"

tools/ci/build_test_apps.sh

@@ -29,6 +29,7 @@ die() {
 [ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
 [ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set"
 [ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set"
+[ -z ${SCAN_CUSTOM_TEST_JSON} ] && die "SCAN_CUSTOM_TEST_JSON is not set"
 [ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
 [ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH}
 
@@ -61,12 +62,9 @@ cd ${IDF_PATH}
 
 # If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
 
-${IDF_PATH}/tools/find_apps.py tools/test_apps \
+${IDF_PATH}/tools/find_apps.py \
     -vv \
     --format json \
-    --build-system cmake \
-    --target ${IDF_TARGET} \
-    --recursive \
     --work-dir "${BUILD_PATH}/@f/@w/@t" \
     --build-dir build \
     --build-log "${LOG_PATH}/@f_@w.txt" \
@@ -74,6 +72,7 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \
     --config 'sdkconfig.ci=default' \
     --config 'sdkconfig.ci.*=' \
     --config '=default' \
+    --app-list ${SCAN_CUSTOM_TEST_JSON}
 
 # --config rules above explained:
 # 1. If sdkconfig.ci exists, use it build the example with configuration name "default"

tools/ci/build_unit_test.sh

@@ -63,7 +63,8 @@ cd ${IDF_PATH}
 # This part of the script produces the same result for all the unit test app build jobs. It may be moved to a separate stage
 # (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
 
-${IDF_PATH}/tools/find_apps.py tools/unit-test-app \
+${IDF_PATH}/tools/find_apps.py \
+    -p tools/unit-test-app \
     -vv \
     --format json \
     --build-system cmake \

tools/ci/check_build_warnings.py

@@ -6,11 +6,11 @@
 # log files for every build.
 # Exits with a non-zero exit code if any warning is found.
 
-import os
-import sys
 import argparse
 import logging
+import os
 import re
+import sys
 
 try:
     from find_build_apps import BuildItem, setup_logging
@@ -70,10 +70,9 @@ def main():
     setup_logging(args)
 
     build_items = [BuildItem.from_json(line) for line in args.build_list]
-
     if not build_items:
-        logging.error("Empty build list!")
+        logging.warning("Empty build list")
-        raise SystemExit(1)
+        SystemExit(0)
 
     found_warnings = 0
     for build_item in build_items:

tools/ci/config/build.yml

@@ -82,6 +82,8 @@ build_esp_idf_tests_cmake_esp32s2:
   artifacts:
     when: always
     expire_in: 4 days
+  variables:
+    SCAN_EXAMPLE_TEST_JSON: ${CI_PROJECT_DIR}/examples/test_configs/scan_${IDF_TARGET}_${EXAMPLE_TEST_BUILD_SYSTEM}.json
   only:
     # Here both 'variables' and 'refs' conditions are given. They are combined with "AND" logic.
     variables:
@@ -96,9 +98,6 @@ build_esp_idf_tests_cmake_esp32s2:
     - mkdir ${BUILD_PATH}
     - mkdir -p ${LOG_PATH}
     - ${IDF_PATH}/tools/ci/build_examples.sh
-    # Check if the tests demand Make built binaries. If not, delete them
-    - if [ ${EXAMPLE_TEST_BUILD_SYSTEM} == "cmake" ]; then exit 0; fi
-    - rm -rf ${BUILD_PATH}
 
 build_examples_make:
   extends: .build_examples_template
@@ -126,6 +125,8 @@ build_examples_make:
 # same as above, but for CMake
 .build_examples_cmake: &build_examples_cmake
   extends: .build_examples_template
+  dependencies:
+    - scan_tests
   artifacts:
     paths:
       - build_examples/list.json
@@ -156,6 +157,8 @@ build_examples_cmake_esp32s2:
 .build_test_apps: &build_test_apps
   extends: .build_template
   stage: build
+  dependencies:
+    - scan_tests
   artifacts:
     when: always
     paths:
@@ -171,8 +174,10 @@ build_examples_cmake_esp32s2:
       - $LOG_PATH
     expire_in: 3 days
   variables:
-    LOG_PATH: "$CI_PROJECT_DIR/log_test_apps"
+    LOG_PATH: "${CI_PROJECT_DIR}/log_test_apps"
-    BUILD_PATH: "$CI_PROJECT_DIR/build_test_apps"
+    BUILD_PATH: "${CI_PROJECT_DIR}/build_test_apps"
+    CUSTOM_TEST_BUILD_SYSTEM: "cmake"
+    SCAN_CUSTOM_TEST_JSON: ${CI_PROJECT_DIR}/tools/test_apps/test_configs/scan_${IDF_TARGET}_${CUSTOM_TEST_BUILD_SYSTEM}.json
   only:
     variables:
       - $BOT_TRIGGER_WITH_LABEL == null

tools/ci/config/pre_check.yml

@@ -191,3 +191,33 @@ check_public_headers:
   script:
     - python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
 
+.scan_build_tests:
+  stage: pre_check
+  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - assign_test
+  variables:
+    CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py
+    TEST_CONFIG_FILE: ${CI_PROJECT_DIR}/tools/ci/config/target-test.yml
+
+scan_tests:
+  extends: .scan_build_tests
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_REGULAR_TEST
+      - $BOT_LABEL_EXAMPLE_TEST
+      - $BOT_LABEL_CUSTOM_TEST
+  artifacts:
+    paths:
+      - $EXAMPLE_TEST_OUTPUT_DIR
+      - $TEST_APPS_OUTPUT_DIR
+  variables:
+    EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples
+    EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs
+    TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
+    TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs
+  script:
+    - python $CI_SCAN_TESTS_PY example_test -b make $EXAMPLE_TEST_DIR --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
+    - python $CI_SCAN_TESTS_PY example_test -b cmake $EXAMPLE_TEST_DIR --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
+    - python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR

tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py

@@ -189,16 +189,15 @@ class AssignTest(object):
         job_list.sort(key=lambda x: x["name"])
         return job_list
 
-    def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None):
+    def search_cases(self, case_filter=None):
         """
-        :param test_case_path: path contains test case folder
         :param case_filter: filter for test cases. the filter to use is default filter updated with case_filter param.
         :return: filtered test case list
         """
         _case_filter = self.DEFAULT_FILTER.copy()
         if case_filter:
             _case_filter.update(case_filter)
-        test_methods = SearchCases.Search.search_test_cases(test_case_path, test_case_file_pattern)
+        test_methods = SearchCases.Search.search_test_cases(self.test_case_path, self.test_case_file_pattern)
         return CaseConfig.filter_test_cases(test_methods, _case_filter)
 
     def _group_cases(self):
@@ -287,7 +286,7 @@ class AssignTest(object):
         failed_to_assign = []
         assigned_groups = []
         case_filter = self._apply_bot_filter()
-        self.test_cases = self._search_cases(self.test_case_path, case_filter, self.test_case_file_pattern)
+        self.test_cases = self.search_cases(case_filter)
         self._apply_bot_test_count()
         test_groups = self._group_cases()
 

tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py

@@ -50,6 +50,7 @@ class Search(object):
 
         for i, test_function in enumerate(test_functions_out):
             print("\t{}. ".format(i + 1) + test_function.case_info["name"])
+            test_function.case_info['app_dir'] = os.path.dirname(file_name)
         return test_functions_out
 
     @classmethod
@@ -124,6 +125,7 @@ class Search(object):
         search all test cases from a folder or file, and then do case replicate.
 
         :param test_case: test case file(s) path
+        :param test_case_file_pattern: unix filename pattern
         :return: a list of replicated test methods
         """
         test_case_files = cls._search_test_case_files(test_case, test_case_file_pattern or cls.TEST_CASE_FILE_PATTERN)

tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py

@@ -139,7 +139,7 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
     def __init__(self, test_case_path, ci_config_file):
         CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group)
 
-    def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None):
+    def search_cases(self, case_filter=None):
         """
         For unit test case, we don't search for test functions.
         The unit test cases is stored in a yaml file which is created in job build-idf-test.
@@ -164,11 +164,11 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
                 return test_cases
 
         test_cases = []
-        if os.path.isdir(test_case_path):
+        if os.path.isdir(self.test_case_path):
-            for yml_file in find_by_suffix('.yml', test_case_path):
+            for yml_file in find_by_suffix('.yml', self.test_case_path):
                 test_cases.extend(get_test_cases_from_yml(yml_file))
-        elif os.path.isfile(test_case_path):
+        elif os.path.isfile(self.test_case_path):
-            test_cases.extend(get_test_cases_from_yml(test_case_path))
+            test_cases.extend(get_test_cases_from_yml(self.test_case_path))
         else:
             print("Test case path is invalid. Should only happen when use @bot to skip unit test.")
 

tools/ci/python_packages/ttfw_idf/CIScanTests.py

@@ -0,0 +1,175 @@
+import argparse
+import errno
+import json
+import logging
+import os
+import re
+from collections import defaultdict
+
+from find_apps import find_apps
+from find_build_apps import BUILD_SYSTEMS, BUILD_SYSTEM_CMAKE
+from ttfw_idf.CIAssignExampleTest import CIExampleAssignTest, TestAppsGroup, ExampleGroup
+
+VALID_TARGETS = [
+    'esp32',
+    'esp32s2',
+]
+
+TEST_LABELS = {
+    'example_test': 'BOT_LABEL_EXAMPLE_TEST',
+    'test_apps': 'BOT_LABEL_CUSTOM_TEST',
+}
+
+BUILD_ALL_LABELS = [
+    'BOT_LABEL_BUILD_ALL_APPS',
+    'BOT_LABEL_REGULAR_TEST',
+]
+
+
+def _has_build_all_label():
+    for label in BUILD_ALL_LABELS:
+        if os.getenv(label):
+            return True
+    return False
+
+
+def _judge_build_or_not(action, build_all):  # type: (str, bool) -> (bool, bool)
+    """
+    :return: (build_or_not_for_test_related_apps, build_or_not_for_non_related_apps)
+    """
+    if build_all or _has_build_all_label() or (not os.getenv('BOT_TRIGGER_WITH_LABEL')):
+        logging.info('Build all apps')
+        return True, True
+
+    if os.getenv(TEST_LABELS[action]):
+        logging.info('Build test cases apps')
+        return True, False
+    else:
+        logging.info('Skip all')
+        return False, False
+
+
+def output_json(apps_dict_list, target, build_system, output_dir):
+    output_path = os.path.join(output_dir, 'scan_{}_{}.json'.format(target.lower(), build_system))
+    with open(output_path, 'w') as fw:
+        fw.writelines([json.dumps(app) + '\n' for app in apps_dict_list])
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Scan the required build tests')
+    parser.add_argument('test_type',
+                        choices=TEST_LABELS.keys(),
+                        help='Scan test type')
+    parser.add_argument('paths',
+                        nargs='+',
+                        help='One or more app paths')
+    parser.add_argument('-b', '--build-system',
+                        choices=BUILD_SYSTEMS.keys(),
+                        default=BUILD_SYSTEM_CMAKE)
+    parser.add_argument('-c', '--ci-config-file',
+                        required=True,
+                        help="gitlab ci config target-test file")
+    parser.add_argument('-o', '--output-path',
+                        required=True,
+                        help="output path of the scan result")
+    parser.add_argument("--exclude",
+                        action="append",
+                        help='Ignore specified directory. Can be used multiple times.')
+    parser.add_argument('--preserve', action="store_true",
+                        help='add this flag to preserve artifacts for all apps')
+    parser.add_argument('--build-all', action="store_true",
+                        help='add this flag to build all apps')
+
+    args = parser.parse_args()
+    build_test_case_apps, build_standalone_apps = _judge_build_or_not(args.test_type, args.build_all)
+
+    if not os.path.exists(args.output_path):
+        try:
+            os.makedirs(args.output_path)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise e
+
+    if (not build_standalone_apps) and (not build_test_case_apps):
+        for target in VALID_TARGETS:
+            output_json([], target, args.build_system, args.output_path)
+            SystemExit(0)
+
+    test_cases = []
+    for path in set(args.paths):
+        if args.test_type == 'example_test':
+            assign = CIExampleAssignTest(path, args.ci_config_file, ExampleGroup)
+        elif args.test_type == 'test_apps':
+            CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+')
+            assign = CIExampleAssignTest(path, args.ci_config_file, TestAppsGroup)
+        else:
+            raise SystemExit(1)  # which is impossible
+
+        test_cases.extend(assign.search_cases())
+
+    '''
+    {
+        <target>: {
+            'test_case_apps': [<app_dir>],   # which is used in target tests
+            'standalone_apps': [<app_dir>],  # which is not
+        },
+        ...
+    }
+    '''
+    scan_info_dict = defaultdict(dict)
+    # store the test cases dir, exclude these folders when scan for standalone apps
+    default_exclude = args.exclude if args.exclude else []
+    exclude_apps = default_exclude
+
+    build_system = args.build_system.lower()
+    build_system_class = BUILD_SYSTEMS[build_system]
+
+    if build_test_case_apps:
+        for target in VALID_TARGETS:
+            target_dict = scan_info_dict[target]
+            test_case_apps = target_dict['test_case_apps'] = set()
+            for case in test_cases:
+                app_dir = case.case_info['app_dir']
+                app_target = case.case_info['target']
+                if app_target.lower() != target.lower():
+                    continue
+                test_case_apps.update(find_apps(build_system_class, app_dir, True, default_exclude, target.lower()))
+                exclude_apps.append(app_dir)
+    else:
+        for target in VALID_TARGETS:
+            scan_info_dict[target]['test_case_apps'] = set()
+
+    if build_standalone_apps:
+        for target in VALID_TARGETS:
+            target_dict = scan_info_dict[target]
+            standalone_apps = target_dict['standalone_apps'] = set()
+            for path in args.paths:
+                standalone_apps.update(find_apps(build_system_class, path, True, exclude_apps, target.lower()))
+    else:
+        for target in VALID_TARGETS:
+            scan_info_dict[target]['standalone_apps'] = set()
+
+    test_case_apps_preserve_default = True if build_system == 'cmake' else False
+    for target in VALID_TARGETS:
+        apps = []
+        for app_dir in scan_info_dict[target]['test_case_apps']:
+            apps.append({
+                'app_dir': app_dir,
+                'build_system': args.build_system,
+                'target': target,
+                'preserve': args.preserve or test_case_apps_preserve_default
+            })
+        for app_dir in scan_info_dict[target]['standalone_apps']:
+            apps.append({
+                'app_dir': app_dir,
+                'build_system': args.build_system,
+                'target': target,
+                'preserve': args.preserve
+            })
+        output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system))
+        with open(output_path, 'w') as fw:
+            fw.writelines([json.dumps(app) + '\n' for app in apps])
+
+
+if __name__ == '__main__':
+    main()

tools/find_apps.py

@@ -5,12 +5,15 @@
 # Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds.
 
 import argparse
-import os
-import sys
-import re
 import glob
+import json
 import logging
+import os
+import re
+import sys
+
 import typing
+
 from find_build_apps import (
     BUILD_SYSTEMS,
     BUILD_SYSTEM_CMAKE,
@@ -22,8 +25,8 @@ from find_build_apps import (
     DEFAULT_TARGET,
 )
 
-# Helper functions
 
+# Helper functions
 
 def dict_from_sdkconfig(path):
     """
@@ -45,9 +48,9 @@ def dict_from_sdkconfig(path):
 # Main logic: enumerating apps and builds
 
 
-def find_builds_for_app(
+def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
-        app_path, work_dir, build_dir, build_log, target_arg, build_system,
+                        build_system, config_rules, preserve_artifacts=True):
-        config_rules):  # type: (str, str, str, str, str, str, typing.List[ConfigRule]) -> typing.List[BuildItem]
+    # type: (str, str, str, str, str, str, typing.List[ConfigRule], bool) -> typing.List[BuildItem]
     """
     Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects
     :param app_path: app directory (can be / usually will be a relative path)
@@ -60,6 +63,7 @@ def find_builds_for_app(
                        a different CONFIG_IDF_TARGET value.
     :param build_system: name of the build system, index into BUILD_SYSTEMS dictionary
     :param config_rules: mapping of sdkconfig file name patterns to configuration names
+    :param preserve_artifacts: determine if the built binary will be uploaded as artifacts.
     :return: list of BuildItems representing build configuration of the app
     """
     build_items = []  # type: typing.List[BuildItem]
@@ -104,6 +108,7 @@ def find_builds_for_app(
                     sdkconfig_path,
                     config_name,
                     build_system,
+                    preserve_artifacts,
                 ))
 
     if not build_items:
@@ -118,14 +123,15 @@ def find_builds_for_app(
                 None,
                 default_config_name,
                 build_system,
+                preserve_artifacts,
             )
         ]
 
     return build_items
 
 
-def find_apps(build_system_class, path, recursive, exclude_list,
+def find_apps(build_system_class, path, recursive, exclude_list, target):
-              target):  # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str) -> typing.List[str]
+    # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str) -> typing.List[str]
     """
     Find app directories in path (possibly recursively), which contain apps for the given build system, compatible
     with the given target.
@@ -189,7 +195,10 @@ def main():
         action="store_true",
         help="Look for apps in the specified directories recursively.",
     )
-    parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE)
+    parser.add_argument(
+        "--build-system",
+        choices=BUILD_SYSTEMS.keys()
+    )
     parser.add_argument(
         "--work-dir",
         help="If set, the app is first copied into the specified directory, and then built." +
@@ -232,12 +241,29 @@ def main():
         type=argparse.FileType("w"),
         help="Output the list of builds to the specified file",
     )
-    parser.add_argument("paths", nargs="+", help="One or more app paths.")
+    parser.add_argument(
+        "--app-list",
+        default=None,
+        help="Scan tests results. Restrict the build/artifacts preservation behavior to apps need to be built. "
+             "If the file does not exist, will build all apps and upload all artifacts."
+    )
+    parser.add_argument(
+        "-p", "--paths",
+        nargs="+",
+        help="One or more app paths."
+    )
     args = parser.parse_args()
     setup_logging(args)
 
-    build_system_class = BUILD_SYSTEMS[args.build_system]
+    # Arguments Validation
-
+    if args.app_list:
+        conflict_args = [args.recursive, args.build_system, args.target, args.exclude, args.paths]
+        if any(conflict_args):
+            raise ValueError('Conflict settings. "recursive", "build_system", "target", "exclude", "paths" should not '
+                             'be specified with "app_list"')
+        if not os.path.exists(args.app_list):
+            raise OSError("File not found {}".format(args.app_list))
+    else:
         # If the build target is not set explicitly, get it from the environment or use the default one (esp32)
         if not args.target:
             env_target = os.environ.get("IDF_TARGET")
@@ -247,37 +273,52 @@ def main():
             else:
                 logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET))
                 args.target = DEFAULT_TARGET
+        if not args.build_system:
+            logging.info("--build-system argument not set, using {} as the default".format(BUILD_SYSTEM_CMAKE))
+            args.build_system = BUILD_SYSTEM_CMAKE
+        required_args = [args.build_system, args.target, args.paths]
+        if not all(required_args):
+            raise ValueError('If app_list not set, arguments "build_system", "target", "paths" are required.')
 
-    # Prepare the list of app paths
+    # Prepare the list of app paths, try to read from the scan_tests result.
-    app_paths = []  # type: typing.List[str]
+    # If the file exists, then follow the file's app_dir and build/artifacts behavior, won't do find_apps() again.
+    # If the file not exists, will do find_apps() first, then build all apps and upload all artifacts.
+    if args.app_list:
+        apps = [json.loads(line) for line in open(args.app_list)]
+    else:
+        app_dirs = []
+        build_system_class = BUILD_SYSTEMS[args.build_system]
         for path in args.paths:
-        app_paths += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target)
+            app_dirs += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target)
+        apps = [{"app_dir": app_dir, "build": True, "preserve": True} for app_dir in app_dirs]
 
-    if not app_paths:
+    if not apps:
-        logging.critical("No {} apps found".format(build_system_class.NAME))
+        logging.warning("No apps found")
-        raise SystemExit(1)
+        SystemExit(0)
-    logging.info("Found {} apps".format(len(app_paths)))
 
-    app_paths = sorted(app_paths)
+    logging.info("Found {} apps".format(len(apps)))
+    apps.sort(key=lambda x: x["app_dir"])
 
     # Find compatible configurations of each app, collect them as BuildItems
     build_items = []  # type: typing.List[BuildItem]
     config_rules = config_rules_from_str(args.config or [])
-    for app_path in app_paths:
+    for app in apps:
         build_items += find_builds_for_app(
-            app_path,
+            app["app_dir"],
             args.work_dir,
             args.build_dir,
             args.build_log,
-            args.target,
+            args.target or app["target"],
-            args.build_system,
+            args.build_system or app["build_system"],
             config_rules,
+            app["preserve"],
         )
     logging.info("Found {} builds".format(len(build_items)))
 
     # Write out the BuildItems. Only JSON supported now (will add YAML later).
     if args.format != "json":
         raise NotImplementedError()
+
     out = args.output or sys.stdout
     out.writelines([item.to_json() + "\n" for item in build_items])
 

tools/find_build_apps/cmake.py

@@ -93,3 +93,33 @@ class CMakeBuildSystem(BuildSystem):
         if CMAKE_PROJECT_LINE not in cmakelists_file_content:
             return False
         return True
+
+    @staticmethod
+    def supported_targets(app_path):
+        formal_to_usual = {
+            'ESP32': 'esp32',
+            'ESP32-S2': 'esp32s2',
+        }
+
+        readme_file_content = BuildSystem._read_readme(app_path)
+        if not readme_file_content:
+            return None
+        match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content)
+        if not match:
+            return None
+        if len(match) > 1:
+            raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path))
+        support_str = match[0].strip()
+
+        targets = []
+        for part in support_str.split('|'):
+            for inner in part.split(' '):
+                inner = inner.strip()
+                if not inner:
+                    continue
+                elif inner in formal_to_usual:
+                    targets.append(formal_to_usual[inner])
+                else:
+                    raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'"
+                                              .format(inner, app_path, ', '.join(formal_to_usual.keys())))
+        return targets

tools/find_build_apps/common.py

@@ -71,6 +71,7 @@ class BuildItem(object):
             sdkconfig_path,
             config_name,
             build_system,
+            preserve_artifacts,
     ):
         # These internal variables store the paths with environment variables and placeholders;
         # Public properties with similar names use the _expand method to get the actual paths.
@@ -84,6 +85,8 @@ class BuildItem(object):
         self.target = target
         self.build_system = build_system
 
+        self.preserve = preserve_artifacts
+
         self._app_name = os.path.basename(os.path.normpath(app_path))
 
         # Some miscellaneous build properties which are set later, at the build stage
@@ -155,6 +158,7 @@ class BuildItem(object):
             "config": self.config_name,
             "target": self.target,
             "verbose": self.verbose,
+            "preserve": self.preserve,
         })
 
     @staticmethod
@@ -172,6 +176,7 @@ class BuildItem(object):
             config_name=d["config"],
             target=d["target"],
             build_system=d["build_system"],
+            preserve_artifacts=d["preserve"]
         )
         result.verbose = d["verbose"]
         return result
@@ -332,34 +337,9 @@ class BuildSystem(object):
             return readme_file.read()
 
     @staticmethod
+    @abstractmethod
     def supported_targets(app_path):
-        formal_to_usual = {
+        pass
-            'ESP32': 'esp32',
-            'ESP32-S2': 'esp32s2',
-        }
-
-        readme_file_content = BuildSystem._read_readme(app_path)
-        if not readme_file_content:
-            return None
-        match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content)
-        if not match:
-            return None
-        if len(match) > 1:
-            raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path))
-        support_str = match[0].strip()
-
-        targets = []
-        for part in support_str.split('|'):
-            for inner in part.split(' '):
-                inner = inner.strip()
-                if not inner:
-                    continue
-                elif inner in formal_to_usual:
-                    targets.append(formal_to_usual[inner])
-                else:
-                    raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'"
-                                              .format(inner, app_path, ', '.join(formal_to_usual.keys())))
-        return targets
 
 
 class BuildError(RuntimeError):

tools/find_build_apps/make.py

@@ -1,8 +1,8 @@
 import logging
 import os
+import shlex
 import subprocess
 import sys
-import shlex
 
 from .common import BuildSystem, BuildError
 
@@ -58,3 +58,7 @@ class MakeBuildSystem(BuildSystem):
         if MAKE_PROJECT_LINE not in makefile_content:
             return False
         return True
+
+    @staticmethod
+    def supported_targets(app_path):
+        return ['esp32']