From 2f7cbd16db3ce7a5095a31cf5ae077ffbbae73aa Mon Sep 17 00:00:00 2001 From: David Cermak Date: Thu, 30 Mar 2023 12:46:20 +0200 Subject: [PATCH 1/2] ci(common): Fix pre-commit manual run --- .github/workflows/pre_commit_check.yml | 16 +++++++++------- .pre-commit-config.yaml | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pre_commit_check.yml b/.github/workflows/pre_commit_check.yml index a2dcb5421..7f2268314 100644 --- a/.github/workflows/pre_commit_check.yml +++ b/.github/workflows/pre_commit_check.yml @@ -12,12 +12,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + with: + ref: ${{ github.base_ref }} - name: Fetch head and base refs - # This is necessary for pre-commit to check the changes in the PR branch + # This is necessary for pre-commit to check the changes in the PR branch (and to set origin/HEAD and HEAD refs) run: | - git fetch origin ${{ github.base_ref }}:base_ref - git fetch origin pull/${{ github.event.pull_request.number }}/head:pr_ref + git fetch origin ${{ github.event.pull_request.head.sha }}:pr_ref + git checkout pr_ref + git remote set-head origin --auto - name: Set up Python environment uses: actions/setup-python@master with: @@ -26,13 +29,12 @@ jobs: run: | pip install pre-commit pre-commit install-hooks - pre-commit install --hook-type commit-msg --hook-type pre-push - name: Run pre-commit and check for any changes run: | echo "Commits being checked:" - git log --oneline --no-decorate base_ref..pr_ref + git log --oneline --no-decorate origin/HEAD..HEAD echo "" - if ! pre-commit run --from-ref base_ref --to-ref pr_ref --show-diff-on-failure ; then + if ! pre-commit run --from-ref origin/HEAD --to-ref HEAD --hook-stage manual --show-diff-on-failure ; then echo "" echo "::notice::It looks like the commits in this PR have been made without having pre-commit hooks installed." echo "::notice::Please see https://github.com/espressif/esp-protocols/CONTRIBUTING.md for instructions." diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9fbc575b4..054daf085 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -57,7 +57,7 @@ repos: hooks: - id: commitizen - id: commitizen-branch - stages: [push] + stages: [push, manual] - repo: local hooks: - id: commit message scopes From 1a0a41fa2d1355215a1590832e7b1c4aa4ed08b6 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Fri, 14 Apr 2023 20:13:12 +0200 Subject: [PATCH 2/2] fix(mdns): Use idf-build-apps package for building mdns --- .github/workflows/mdns__build-target-test.yml | 121 +++--------------- .github/workflows/modem__build-host-tests.yml | 2 +- ci/build_apps.py | 16 +-- components/mdns/.build-test-rules.yml | 11 ++ components/mdns/tests/unit_test/sdkconfig.ci | 3 + 5 files changed, 38 insertions(+), 115 deletions(-) create mode 100644 components/mdns/.build-test-rules.yml create mode 100644 components/mdns/tests/unit_test/sdkconfig.ci diff --git a/.github/workflows/mdns__build-target-test.yml b/.github/workflows/mdns__build-target-test.yml index 2f73a16ef..2baf99a0c 100644 --- a/.github/workflows/mdns__build-target-test.yml +++ b/.github/workflows/mdns__build-target-test.yml @@ -13,122 +13,33 @@ jobs: name: Build strategy: matrix: - idf_ver: ["latest"] - idf_target: ["esp32", "esp32s2", "esp32c3"] - test: [ { app: example, path: "components/mdns/examples" }, { app: unit_test, path: "components/mdns/tests/unit_test" } ] + idf_ver: ["latest", "release-v5.0"] + test: [ { app: example, path: "examples" }, { app: unit_test, path: "tests/unit_test" }, { app: test_app, path: "tests/test_apps" } ] runs-on: ubuntu-20.04 container: espressif/idf:${{ matrix.idf_ver }} steps: - name: Checkout esp-protocols uses: actions/checkout@v3 - with: - submodules: recursive - - name: Build ${{ matrix.test.app }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} + - name: Build ${{ matrix.test.app }} with IDF-${{ matrix.idf_ver }} shell: bash - working-directory: ${{ matrix.test.path }} run: | - ${IDF_PATH}/install.sh --enable-pytest . ${IDF_PATH}/export.sh - python $IDF_PATH/tools/ci/ci_build_apps.py . --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app - for dir in `ls -d build_*`; do + python -m pip install idf-build-apps + # Build default configs for all targets + python ./ci/build_apps.py components/mdns/${{ matrix.test.path }} -r default -m components/mdns/.build-test-rules.yml -d + # Build specific configs for test targets + python ./ci/build_apps.py components/mdns/${{ matrix.test.path }} + cd components/mdns/${{ matrix.test.path }} + for dir in `ls -d build_esp32_*`; do $GITHUB_WORKSPACE/ci/clean_build_artifacts.sh `pwd`/$dir zip -qur artifacts.zip $dir done - uses: actions/upload-artifact@v3 - if: ${{ matrix.idf_target }} == "esp32" with: - name: mdns_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.test.app }} - path: ${{ matrix.test.path }}/artifacts.zip + name: mdns_bin_esp32_${{ matrix.idf_ver }}_${{ matrix.test.app }} + path: components/mdns/${{ matrix.test.path }}/artifacts.zip if-no-files-found: error - build_mdns_app: - if: contains(github.event.pull_request.labels.*.name, 'mdns') || github.event_name == 'push' - name: Build Test Apps - strategy: - matrix: - idf_ver: ["latest"] - idf_target: ["esp32", "esp32s2", "esp32c3"] - runs-on: ubuntu-20.04 - container: espressif/idf:${{ matrix.idf_ver }} - steps: - - name: Checkout esp-protocols - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Setup Build Environment - working-directory: components/mdns/tests/test_apps - run: | - . ${IDF_PATH}/export.sh - pip install -r $GITHUB_WORKSPACE/ci/requirements.txt - - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} - env: - IDF_TARGET: ${{ matrix.idf_target }} - shell: bash - working-directory: components/mdns/tests/test_apps - run: | - . ${IDF_PATH}/export.sh - python $IDF_PATH/tools/ci/ci_build_apps.py . --target ${{ matrix.idf_target }} -vv --pytest-apps - - name: Merge binaries with IDF-${{ matrix.idf_ver }} - working-directory: components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default - env: - IDF_TARGET: ${{ matrix.idf_target }} - shell: bash - run: | - . ${IDF_PATH}/export.sh - esptool.py --chip ${{ matrix.idf_target }} merge_bin --fill-flash-size 4MB -o flash_image.bin @flash_args - - uses: actions/upload-artifact@v2 - with: - name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }} - path: | - components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/bootloader/bootloader.bin - components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/partition_table/partition-table.bin - components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/*.bin - components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/*.elf - components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/flasher_args.json - components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/config/sdkconfig.h - components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/config/sdkconfig.json - if-no-files-found: error - - target_test_apps_mdns: - # Skip running on forks since it won't have access to secrets - if: | - github.repository == 'espressif/esp-protocols' && - ( contains(github.event.pull_request.labels.*.name, 'mdns') || github.event_name == 'push' ) - name: Target Test Apps - strategy: - matrix: - idf_ver: ["latest"] - idf_target: ["esp32"] - needs: build_mdns_app - runs-on: - - self-hosted - - ESP32-ETHERNET-KIT - steps: - - name: Clear repository - run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v2 - with: - name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }} - path: components/mdns/tests/test_apps/build - - name: Install Python packages - env: - PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" - run: | - sudo apt-get install -y dnsutils - - name: Download Test apps to target - run: | - python -m esptool --chip ${{ matrix.idf_target }} write_flash 0x0 components/mdns/tests/test_apps/build/flash_image.bin - - name: Run Example Test on target - working-directory: components/mdns/tests/test_apps - run: | - python -m pytest --log-cli-level DEBUG --junit-xml=./examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}.xml --target=${{matrix.idf_target}} - - uses: actions/upload-artifact@v2 - if: always() - with: - name: test_apps_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }} - path: components/mdns/tests/test_apps/*.xml - target_tests_mdns: # Skip running on forks since it won't have access to secrets if: | @@ -139,7 +50,7 @@ jobs: matrix: idf_ver: ["latest"] idf_target: ["esp32"] - test: [ { app: example, path: "components/mdns/examples" }, { app: unit_test, path: "components/mdns/tests/unit_test" } ] + test: [ { app: example, path: "examples" }, { app: unit_test, path: "tests/unit_test" }, { app: test_app, path: "tests/test_apps" } ] needs: build_mdns runs-on: - self-hosted @@ -151,14 +62,14 @@ jobs: - uses: actions/download-artifact@v3 with: name: mdns_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.test.app }} - path: ${{ matrix.test.path }}/ci/ + path: components/mdns/${{ matrix.test.path }}/ci/ - name: Install Python packages env: PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" run: | sudo apt-get install -y dnsutils - name: Run ${{ matrix.test.app }} application on ${{ matrix.idf_target }} - working-directory: ${{ matrix.test.path }} + working-directory: components/mdns/${{ matrix.test.path }} run: | unzip ci/artifacts.zip -d ci for dir in `ls -d ci/build_*`; do @@ -170,4 +81,4 @@ jobs: if: always() with: name: results_${{ matrix.test.app }}_${{ matrix.idf_target }}_${{ matrix.idf_ver }}.xml - path: ${{ matrix.test.path }}/*.xml + path: components/mdns/${{ matrix.test.path }}/*.xml diff --git a/.github/workflows/modem__build-host-tests.yml b/.github/workflows/modem__build-host-tests.yml index 79a8d992f..009d032b9 100644 --- a/.github/workflows/modem__build-host-tests.yml +++ b/.github/workflows/modem__build-host-tests.yml @@ -48,7 +48,7 @@ jobs: . ${IDF_PATH}/export.sh python -m pip install idf-build-apps cd $GITHUB_WORKSPACE/protocols - python ./ci/build_apps.py components/esp_modem/examples/${{ matrix.example }} + python ./ci/build_apps.py components/esp_modem/examples/${{ matrix.example }} -m components/esp_modem/examples/.build-test-rules.yml host_test_esp_modem: if: contains(github.event.pull_request.labels.*.name, 'modem') || github.event_name == 'push' diff --git a/ci/build_apps.py b/ci/build_apps.py index a6e577977..d7aa2e0e6 100644 --- a/ci/build_apps.py +++ b/ci/build_apps.py @@ -7,7 +7,6 @@ This file is used in CI for esp-protocols build tests import argparse import os import sys -from pathlib import Path from idf_build_apps import build_apps, find_apps, setup_logging from idf_build_apps.constants import SUPPORTED_TARGETS @@ -24,6 +23,9 @@ if __name__ == '__main__': default='all', help='Build apps for given target', ) + parser.add_argument('-r', '--rules', nargs='*', default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'], help='Rules how to treat configs') + parser.add_argument('-m', '--manifests', nargs='*', default=[], help='list of manifest files') + parser.add_argument('-d', '--delete', action='store_true', help='Delete build artifacts') args = parser.parse_args() IDF_PATH = os.environ['IDF_PATH'] @@ -32,19 +34,15 @@ if __name__ == '__main__': setup_logging(2) apps = find_apps( args.paths, - recursive=True, + recursive=False, target=args.target, build_dir='build_@t_@w', - config_rules_str=[ - 'sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default' - ], + config_rules_str=args.rules, build_log_path='build_log.txt', size_json_path='size.json', check_warnings=True, - preserve=True, - manifest_files=[ - str(p) for p in Path('.').glob('**/.build-test-rules.yml') - ], + preserve=not args.delete, + manifest_files=args.manifests, default_build_targets=SUPPORTED_TARGETS, manifest_rootpath='.', ) diff --git a/components/mdns/.build-test-rules.yml b/components/mdns/.build-test-rules.yml new file mode 100644 index 000000000..9a4da23d2 --- /dev/null +++ b/components/mdns/.build-test-rules.yml @@ -0,0 +1,11 @@ +components/mdns/examples: + disable: + - if: IDF_TARGET in ["esp32h2"] + +components/mdns/tests/unit_test: + disable: + - if: IDF_TARGET in ["esp32h2"] + +components/mdns/tests/test_apps: + disable: + - if: IDF_TARGET in ["esp32h2"] diff --git a/components/mdns/tests/unit_test/sdkconfig.ci b/components/mdns/tests/unit_test/sdkconfig.ci new file mode 100644 index 000000000..966071918 --- /dev/null +++ b/components/mdns/tests/unit_test/sdkconfig.ci @@ -0,0 +1,3 @@ +CONFIG_IDF_TARGET="esp32" +CONFIG_UNITY_ENABLE_FIXTURE=y +CONFIG_UNITY_ENABLE_IDF_TEST_RUNNER=n