espressif/esp-idf
1
0
Fork 1
mirror of https://github.com/espressif/esp-idf.git synced 2025-07-31 19:24:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

update after MR reviews

Browse Source
This commit is contained in:
Fu Hanxi
2020-06-28 13:56:04 +08:00
parent a882353c32
commit 270e7dec38
9 changed files with 92 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tools/build_apps.py
View File

@@ -5,9 +5,10 @@
#
import argparse
import logging
import shutil
import sys
import logging
from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS
@@ -109,7 +110,7 @@ def main():
failed_builds = []
for build_info in builds_for_current_job:
if not build_info.build:
logging.info('Skip build detected. Skipping...')
logging.info("Skip building app {}".format(build_info.app_dir))
continue
logging.info("Running build {}: {}".format(build_info.index, repr(build_info)))
@@ -124,9 +125,9 @@ def main():
raise SystemExit(1)
else:
if not build_info.preserve:
logging.info('NOT preserve artifacts detected. Deleting...')
logging.info("Removing build directory {}".format(build_info.build_dir))
# we only remove binaries here, log files are still needed by check_build_warnings.py
shutil.rmtree(build_info.work_dir, ignore_errors=True)
shutil.rmtree(build_info.build_dir, ignore_errors=True)
if failed_builds:
logging.error("The following build have failed:")

8
tools/ci/build_examples.sh
View File

@@ -72,13 +72,9 @@ cd ${IDF_PATH}
# If changing the work-dir or build-dir format, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
${IDF_PATH}/tools/find_apps.py examples \
${IDF_PATH}/tools/find_apps.py \
-vv \
--format json \
--build-system ${EXAMPLE_TEST_BUILD_SYSTEM} \
--target ${IDF_TARGET} \
--recursive \
--exclude examples/build_system/idf_as_lib \
--work-dir "${BUILD_PATH}/@f/@w/@t" \
--build-dir build \
--build-log "${LOG_PATH}/@f_@w.txt" \
@@ -86,7 +82,7 @@ ${IDF_PATH}/tools/find_apps.py examples \
--config 'sdkconfig.ci=default' \
--config 'sdkconfig.ci.*=' \
--config '=default' \
--scan-tests-json ${SCAN_EXAMPLE_TEST_JSON}
--app-list ${SCAN_EXAMPLE_TEST_JSON}
# --config rules above explained:
# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"

7
tools/ci/build_test_apps.sh
View File

@@ -62,12 +62,9 @@ cd ${IDF_PATH}
# If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
${IDF_PATH}/tools/find_apps.py tools/test_apps \
${IDF_PATH}/tools/find_apps.py \
-vv \
--format json \
--build-system cmake \
--target ${IDF_TARGET} \
--recursive \
--work-dir "${BUILD_PATH}/@f/@w/@t" \
--build-dir build \
--build-log "${LOG_PATH}/@f_@w.txt" \
@@ -75,7 +72,7 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \
--config 'sdkconfig.ci=default' \
--config 'sdkconfig.ci.*=' \
--config '=default' \
--scan-tests-json ${SCAN_CUSTOM_TEST_JSON}
--app-list ${SCAN_CUSTOM_TEST_JSON}
# --config rules above explained:
# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"

3
tools/ci/build_unit_test.sh
View File

@@ -63,7 +63,8 @@ cd ${IDF_PATH}
# This part of the script produces the same result for all the unit test app build jobs. It may be moved to a separate stage
# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
${IDF_PATH}/tools/find_apps.py tools/unit-test-app \
${IDF_PATH}/tools/find_apps.py \
-p tools/unit-test-app \
-vv \
--format json \
--build-system cmake \

6
tools/ci/check_build_warnings.py
View File

@@ -6,11 +6,11 @@
# log files for every build.
# Exits with a non-zero exit code if any warning is found.
import os
import sys
import argparse
import logging
import os
import re
import sys
try:
from find_build_apps import BuildItem, setup_logging
@@ -78,7 +78,7 @@ def main():
found_warnings = 0
for build_item in build_items:
if not build_item.build:
logging.debug('Skipping build detected. Skipping checking...')
logging.debug("Skip checking build log for app {}".format(build_item.app_dir))
continue
if not build_item.build_log_path:
logging.debug("No log file for {}".format(build_item.work_dir))

3
tools/ci/config/build.yml
View File

@@ -176,7 +176,8 @@ build_examples_cmake_esp32s2:
variables:
LOG_PATH: "${CI_PROJECT_DIR}/log_test_apps"
BUILD_PATH: "${CI_PROJECT_DIR}/build_test_apps"
SCAN_CUSTOM_TEST_JSON: ${CI_PROJECT_DIR}/tools/test_apps/test_configs/scan_${IDF_TARGET}.json
CUSTOM_TEST_BUILD_SYSTEM: "cmake"
SCAN_CUSTOM_TEST_JSON: ${CI_PROJECT_DIR}/tools/test_apps/test_configs/scan_${IDF_TARGET}_${CUSTOM_TEST_BUILD_SYSTEM}.json
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null

5
tools/ci/config/pre_check.yml
View File

@@ -203,6 +203,7 @@ scan_tests:
extends: .scan_build_tests
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_EXAMPLE_TEST
- $BOT_LABEL_CUSTOM_TEST
artifacts:
@@ -215,6 +216,6 @@ scan_tests:
TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs
script:
- python $CI_SCAN_TESTS_PY example_test -b make $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
- python $CI_SCAN_TESTS_PY example_test -b cmake $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
- python $CI_SCAN_TESTS_PY example_test -b make $EXAMPLE_TEST_DIR --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
- python $CI_SCAN_TESTS_PY example_test -b cmake $EXAMPLE_TEST_DIR --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
- python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR

30
tools/ci/python_packages/ttfw_idf/CIScanTests.py
View File

@@ -52,13 +52,21 @@ def main():
actions = parser.add_subparsers(dest='action')
common = argparse.ArgumentParser(add_help=False)
common.add_argument('paths', type=str, nargs='+',
common.add_argument('paths',
nargs='+',
help="One or more app paths")
common.add_argument('-b', '--build-system', choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE)
common.add_argument('-c', '--ci-config-file', type=str, required=True,
common.add_argument('-b', '--build-system',
choices=BUILD_SYSTEMS.keys(),
default=BUILD_SYSTEM_CMAKE)
common.add_argument('-c', '--ci-config-file',
required=True,
help="gitlab ci config target-test file")
common.add_argument('-o', '--output-path', type=str, required=True,
common.add_argument('-o', '--output-path',
required=True,
help="output path of the scan result")
common.add_argument("--exclude",
action="append",
help="Ignore specified directory. Can be used multiple times.")
common.add_argument('--preserve', action="store_true",
help='add this flag to preserve artifacts for all apps')
common.add_argument('--build-all', action="store_true",
@@ -99,7 +107,8 @@ def main():
'''
scan_info_dict = defaultdict(dict)
# store the test cases dir, exclude these folders when scan for standalone apps
exclude_apps = []
default_exclude = args.exclude if args.exclude else []
exclude_apps = default_exclude
build_system = args.build_system.lower()
build_system_class = BUILD_SYSTEMS[build_system]
@@ -112,7 +121,7 @@ def main():
app_target = case.case_info['target']
if app_target.lower() != target.lower():
continue
test_case_apps.update(find_apps(build_system_class, app_dir, True, [], target.lower()))
test_case_apps.update(find_apps(build_system_class, app_dir, True, default_exclude, target.lower()))
exclude_apps.append(app_dir)
for target in VALID_TARGETS:
@@ -130,18 +139,21 @@ def main():
apps.append({
'app_dir': app_dir,
'build': True,
'build_system': args.build_system,
'target': target,
'preserve': args.preserve or test_case_apps_preserve_default
})
for app_dir in scan_info_dict[target]['standalone_apps']:
apps.append({
'app_dir': app_dir,
'build': build_all if build_system == 'cmake' else True,
'build_system': args.build_system,
'target': target,
'preserve': (args.preserve and build_all) if build_system == 'cmake' else False
})
output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system))
if apps:
with open(output_path, 'w') as fw:
fw.writelines([json.dumps(app) + '\n' for app in apps])
with open(output_path, 'w') as fw:
fw.writelines([json.dumps(app) + '\n' for app in apps])
if __name__ == '__main__':

83
tools/find_apps.py
View File

@@ -5,12 +5,13 @@
# Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds.
import argparse
import json
import os
import sys
import re
import glob
import json
import logging
import os
import re
import sys
import typing
from find_build_apps import (
@@ -197,7 +198,10 @@ def main():
action="store_true",
help="Look for apps in the specified directories recursively.",
)
parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE)
parser.add_argument(
"--build-system",
choices=BUILD_SYSTEMS.keys()
)
parser.add_argument(
"--work-dir",
help="If set, the app is first copied into the specified directory, and then built." +
@@ -241,60 +245,77 @@ def main():
help="Output the list of builds to the specified file",
)
parser.add_argument(
'-s',
'--scan-tests-json',
"--app-list",
default=None,
help="Scan tests result. Restrict the build/architect behavior to apps need to be built.\n"
help="Scan tests results. Restrict the build/artifacts preservation behavior to apps need to be built. "
"If the file does not exist, will build all apps and upload all artifacts."
)
parser.add_argument("paths", nargs="+", help="One or more app paths.")
parser.add_argument(
"-p", "--paths",
nargs="+",
help="One or more app paths."
)
args = parser.parse_args()
setup_logging(args)
build_system_class = BUILD_SYSTEMS[args.build_system]
# If the build target is not set explicitly, get it from the environment or use the default one (esp32)
if not args.target:
env_target = os.environ.get("IDF_TARGET")
if env_target:
logging.info("--target argument not set, using IDF_TARGET={} from the environment".format(env_target))
args.target = env_target
else:
logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET))
args.target = DEFAULT_TARGET
# Arguments Validation
if args.app_list:
conflict_args = [args.recursive, args.build_system, args.target, args.exclude, args.paths]
if any(conflict_args):
raise ValueError('Conflict settings. "recursive", "build_system", "target", "exclude", "paths" should not '
'be specified with "app_list"')
if not os.path.exists(args.app_list):
raise OSError("File not found {}".format(args.app_list))
else:
# If the build target is not set explicitly, get it from the environment or use the default one (esp32)
if not args.target:
env_target = os.environ.get("IDF_TARGET")
if env_target:
logging.info("--target argument not set, using IDF_TARGET={} from the environment".format(env_target))
args.target = env_target
else:
logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET))
args.target = DEFAULT_TARGET
if not args.build_system:
logging.info("--build-system argument not set, using {} as the default".format(BUILD_SYSTEM_CMAKE))
args.build_system = BUILD_SYSTEM_CMAKE
required_args = [args.build_system, args.target, args.paths]
if not all(required_args):
raise ValueError('If app_list not set, arguments "build_system", "target", "paths" are required.')
# Prepare the list of app paths, try to read from the scan_tests result.
# If the file exists, then follow the file's app_dir and build/archifacts behavior, won't do find_apps() again.
# If the file exists, then follow the file's app_dir and build/artifacts behavior, won't do find_apps() again.
# If the file not exists, will do find_apps() first, then build all apps and upload all artifacts.
if args.scan_tests_json and os.path.exists(args.scan_tests_json):
apps = [json.loads(line) for line in open(args.scan_tests_json)]
if args.app_list:
apps = [json.loads(line) for line in open(args.app_list)]
else:
app_dirs = []
build_system_class = BUILD_SYSTEMS[args.build_system]
for path in args.paths:
app_dirs += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target)
apps = [{'app_dir': app_dir, 'build': True, 'preserve': True} for app_dir in app_dirs]
apps = [{"app_dir": app_dir, "build": True, "preserve": True} for app_dir in app_dirs]
if not apps:
logging.critical("No {} apps found".format(build_system_class.NAME))
logging.critical("No apps found")
raise SystemExit(1)
logging.info("Found {} apps".format(len(apps)))
apps.sort(key=lambda x: x['app_dir'])
apps.sort(key=lambda x: x["app_dir"])
# Find compatible configurations of each app, collect them as BuildItems
build_items = [] # type: typing.List[BuildItem]
config_rules = config_rules_from_str(args.config or [])
for app in apps:
build_items += find_builds_for_app(
app['app_dir'],
app["app_dir"],
args.work_dir,
args.build_dir,
args.build_log,
args.target,
args.build_system,
args.target or app["target"],
args.build_system or app["build_system"],
config_rules,
app['build'],
app['preserve'],
app["build"],
app["preserve"],
)
logging.info("Found {} builds".format(len(build_items)))