From f29719fa0b5e1bfb7908b0bad78a46d82d64f8ba Mon Sep 17 00:00:00 2001 From: David Cermak Date: Tue, 14 Mar 2023 11:34:02 +0100 Subject: [PATCH 1/4] ci(common): force scoping commit messages with components Add simple changelog generator and mdns tag formatter --- .pre-commit-config.yaml | 10 ++++ ci/changelog.py | 98 ++++++++++++++++++++++++++++++++++++++++ components/mdns/.cz.yaml | 8 ++++ 3 files changed, 116 insertions(+) create mode 100644 ci/changelog.py create mode 100644 components/mdns/.cz.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3def3ffa7..9fbc575b4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,7 @@ repos: rev: 5.0.4 hooks: - id: flake8 + args: ['--config=.flake8', '--tee', '--benchmark'] - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.981 hooks: @@ -32,6 +33,7 @@ repos: rev: "v0.32.0" hooks: - id: yapf + args: ['style={based_on_style: google, column_limit: 160, indent_width: 4}'] - repo: https://github.com/pre-commit/mirrors-isort rev: v5.10.1 hooks: @@ -56,3 +58,11 @@ repos: - id: commitizen - id: commitizen-branch stages: [push] + - repo: local + hooks: + - id: commit message scopes + name: "commit message must be scoped with: mdns, modem, websocket, asio, mqtt_cxx, common" + entry: '\A(?!(feat|fix|ci|bump|test|docs)\((mdns|modem|common|websocket|asio|mqtt_cxx)\)\:)' + language: pygrep + args: [--multiline] + stages: [commit-msg] diff --git a/ci/changelog.py b/ci/changelog.py new file mode 100644 index 000000000..d4f42471d --- /dev/null +++ b/ci/changelog.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import argparse +import os +import re + +import sh + + +def main(): + parser = argparse.ArgumentParser( + description='Generates a change log from cz commits') + parser.add_argument('component') + args = parser.parse_args() + + old_ref = os.environ.get('CZ_PRE_CURRENT_TAG_VERSION') + new_tag = os.environ.get('CZ_PRE_NEW_TAG_VERSION') + new_ref = os.environ.get('CZ_PRE_NEW_VERSION') + component = args.component + + root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + + git = sh.git.bake(_cwd=root_path) + + keys = ['breaking', 'major', 'feat', 'fix', 'update'] + items = {key: [] for key in keys} + sections = { + 'feat': 'Features', + 'fix': 'Bug Fixes', + 'update': 'Updated', + 'breaking': 'Breaking changes', + 'major': 'Major changes' + } + brief_log = git.log('--oneline', '{}..HEAD'.format(old_ref), '--', 'components/' + component, _tty_out=False) + for oneline in brief_log.splitlines(): + [commit, brief_msg] = oneline.split(' ', 1) + if 'Merge' in str(brief_msg) or brief_msg.startswith('bump('): # skip Merge commits and bumps + continue + find_title = re.findall(r'^(fix|feat|esp_modem|esp-modem|mdns)(\([^\)]+\))?:\s*(.+)$', str(brief_msg)) + item_type = '' # default prefix + item_message = brief_msg # default body + if find_title is not None and len(find_title) > 0: # Use scopes & types + item_type = find_title[0][0] # prefix (fix/feat/ci) + item_message = find_title[0][-1] # title body + + # Add details in parentheses (commit-id and references from the FOOTER section) + details = '[{}](https://github.com/espressif/esp-protocols/commit/{})'.format(commit, commit) + msg_details = git.show('-s', commit, _tty_out=False) + # check references + if any(str(msg_details) in s for s in ['esp-protocols/issues', 'BREAKING CHANGE', 'MAJOR CHANGE']): + # Closes + closes = re.findall(r'Closes ([^\d]+/(\d+))', str(msg_details), re.MULTILINE) + if closes and closes[0] is not None: + details += ', [#{}]({})'.format(closes[0][1], closes[0][0]) + # Breaking changes + find_breaking = re.findall(r'BREAKING CHANGE:\s*([^\n]*)', str(msg_details), re.MULTILINE) + if find_breaking is not None and len(find_breaking) > 0: + if find_breaking[0] != '': + items['breaking'].append('{} ([{}](https://github.com/espressif/esp-protocols/commit/{}))'.format(find_breaking[0], item_message, commit)) + else: + items['breaking'].append('{} ({})'.format(item_message, details)) + details += ', !BREAKING' + # Major changes + find_major = re.findall(r'MAJOR CHANGE:\s*([^\n]*)', + str(msg_details), re.MULTILINE) + if find_major is not None and len( + find_major) > 0 and find_major[0] != '': + items['major'].append('{} ([{}](https://github.com/espressif/esp-protocols/commit/{}))'.format(find_major[0], item_message, commit)) + if item_type in ['fix', 'feat']: + items[item_type].append('{} ({})'.format(item_message, details)) + else: + items['update'].append('{} ({})'.format(item_message, details)) + # Generate changelog + changelog = '## [{}](https://github.com/espressif/esp-protocols/commits/{})\n'.format( + new_ref, new_tag) + for section, item in items.items(): + if len(item) > 0: + changelog += '\n### {}\n\n'.format(sections[section]) + for it in item: + changelog += '- {}\n'.format(it) + changelog += '\n' + filename = os.path.join(root_path, 'components', component, 'CHANGELOG.md') + # insert the actual changelog to the beginning of the file, just after the title (2nd line) + with open(filename, 'r') as orig_changelog: + changelog_title = orig_changelog.readline( + ) # expect # Changelog title on the first line + changelog_nl = orig_changelog.readline() + orig_items = orig_changelog.read() + with open(filename, 'w') as updated_changelog: + updated_changelog.write(changelog_title) + updated_changelog.write(changelog_nl) + updated_changelog.write(changelog) + updated_changelog.write(orig_items) + git.add(filename) + + +if __name__ == '__main__': + main() diff --git a/components/mdns/.cz.yaml b/components/mdns/.cz.yaml new file mode 100644 index 000000000..9ce402527 --- /dev/null +++ b/components/mdns/.cz.yaml @@ -0,0 +1,8 @@ +--- +commitizen: + bump_message: 'bump(mdns): $current_version -> $new_version' + pre_bump_hooks: python ../../ci/changelog.py mdns + tag_format: mdns-v$version + version: 1.0.8 + version_files: + - idf_component.yml From c92ce075fd2d3fa4828edfb4aced2d413327a86c Mon Sep 17 00:00:00 2001 From: David Cermak Date: Wed, 15 Mar 2023 16:57:01 +0100 Subject: [PATCH 2/4] bump(mdns): 1.0.8 -> 1.0.9 --- components/mdns/.cz.yaml | 2 +- components/mdns/CHANGELOG.md | 19 +++++++++++++++++++ components/mdns/idf_component.yml | 2 +- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/components/mdns/.cz.yaml b/components/mdns/.cz.yaml index 9ce402527..25b615a59 100644 --- a/components/mdns/.cz.yaml +++ b/components/mdns/.cz.yaml @@ -3,6 +3,6 @@ commitizen: bump_message: 'bump(mdns): $current_version -> $new_version' pre_bump_hooks: python ../../ci/changelog.py mdns tag_format: mdns-v$version - version: 1.0.8 + version: 1.0.9 version_files: - idf_component.yml diff --git a/components/mdns/CHANGELOG.md b/components/mdns/CHANGELOG.md index 593fb8a87..21ae97de9 100644 --- a/components/mdns/CHANGELOG.md +++ b/components/mdns/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## [1.0.9](https://github.com/espressif/esp-protocols/commits/mdns-v1.0.9) + +### Features + +- Add reverse lookup to the example and test ([d464ed7](https://github.com/espressif/esp-protocols/commit/d464ed7)) +- Add support for IPv6 reverse query ([d4825f5](https://github.com/espressif/esp-protocols/commit/d4825f5)) + +### Bug Fixes + +- Reintroduce missing CHANGELOGs ([200cbb3](https://github.com/espressif/esp-protocols/commit/200cbb3)) +- use semaphore instead of task notification bits (IDFGH-9380) ([73f2800](https://github.com/espressif/esp-protocols/commit/73f2800), [IDF#10754](https://github.com/espressif/esp-idf/issues/10754)) + +### Updated + +- ci(common): force scoping commit messages with components ([c55fcc0](https://github.com/espressif/esp-protocols/commit/c55fcc0)) +- Add homepage URL and License to all components ([ef3f0ee](https://github.com/espressif/esp-protocols/commit/ef3f0ee)) +- docs: fix of mdns link translation ([1c850dd](https://github.com/espressif/esp-protocols/commit/1c850dd)) +- unite all tags under common structure py test: update tags under common structure ([c6db3ea](https://github.com/espressif/esp-protocols/commit/c6db3ea)) + ## [1.0.8](https://github.com/espressif/esp-protocols/commits/b9b4a75) ### Features diff --git a/components/mdns/idf_component.yml b/components/mdns/idf_component.yml index 7a62ba78e..fe43cddbf 100644 --- a/components/mdns/idf_component.yml +++ b/components/mdns/idf_component.yml @@ -1,4 +1,4 @@ -version: "1.0.8" +version: "1.0.9" description: mDNS url: https://github.com/espressif/esp-protocols/tree/master/components/mdns dependencies: From 4c7720a2c113da9d2914d898c4ccf59ae0471393 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Thu, 16 Mar 2023 08:31:18 +0100 Subject: [PATCH 3/4] ci(common): Run all test on push event Added a description about the release process to the contributing guide --- .github/workflows/asio__build-target-test.yml | 6 ++++-- .github/workflows/mdns__build-target-test.yml | 12 ++++++++---- .github/workflows/mdns__host-tests.yml | 4 ++-- .github/workflows/modem__build-host-tests.yml | 6 +++--- .github/workflows/modem__target-test.yml | 6 ++++-- .github/workflows/mqtt_cxx__build.yml | 2 +- .../workflows/websocket__build-target-test.yml | 6 ++++-- CONTRIBUTING.md | 16 ++++++++++++++++ 8 files changed, 42 insertions(+), 16 deletions(-) diff --git a/.github/workflows/asio__build-target-test.yml b/.github/workflows/asio__build-target-test.yml index 74d12ad99..c6bd2cda3 100644 --- a/.github/workflows/asio__build-target-test.yml +++ b/.github/workflows/asio__build-target-test.yml @@ -9,7 +9,7 @@ on: jobs: build_asio: - if: contains(github.event.pull_request.labels.*.name, 'asio') + if: contains(github.event.pull_request.labels.*.name, 'asio') || github.event_name == 'push' name: Build strategy: matrix: @@ -58,7 +58,9 @@ jobs: target_tests_asio: # Skip running on forks since it won't have access to secrets - if: ${{ contains(github.event.pull_request.labels.*.name, 'asio') && (github.repository == 'espressif/esp-protocols') }} + if: | + github.repository == 'espressif/esp-protocols' && + ( contains(github.event.pull_request.labels.*.name, 'asio') || github.event_name == 'push' ) name: Target tests strategy: matrix: diff --git a/.github/workflows/mdns__build-target-test.yml b/.github/workflows/mdns__build-target-test.yml index d5f76171d..2f73a16ef 100644 --- a/.github/workflows/mdns__build-target-test.yml +++ b/.github/workflows/mdns__build-target-test.yml @@ -9,7 +9,7 @@ on: jobs: build_mdns: - if: contains(github.event.pull_request.labels.*.name, 'mdns') + if: contains(github.event.pull_request.labels.*.name, 'mdns') || github.event_name == 'push' name: Build strategy: matrix: @@ -42,7 +42,7 @@ jobs: if-no-files-found: error build_mdns_app: - if: contains(github.event.pull_request.labels.*.name, 'mdns') + if: contains(github.event.pull_request.labels.*.name, 'mdns') || github.event_name == 'push' name: Build Test Apps strategy: matrix: @@ -91,7 +91,9 @@ jobs: target_test_apps_mdns: # Skip running on forks since it won't have access to secrets - if: ${{ contains(github.event.pull_request.labels.*.name, 'mdns') && (github.repository == 'espressif/esp-protocols') }} + if: | + github.repository == 'espressif/esp-protocols' && + ( contains(github.event.pull_request.labels.*.name, 'mdns') || github.event_name == 'push' ) name: Target Test Apps strategy: matrix: @@ -129,7 +131,9 @@ jobs: target_tests_mdns: # Skip running on forks since it won't have access to secrets - if: ${{ contains(github.event.pull_request.labels.*.name, 'mdns') && (github.repository == 'espressif/esp-protocols') }} + if: | + github.repository == 'espressif/esp-protocols' && + ( contains(github.event.pull_request.labels.*.name, 'mdns') || github.event_name == 'push' ) name: Target Example and Unit tests strategy: matrix: diff --git a/.github/workflows/mdns__host-tests.yml b/.github/workflows/mdns__host-tests.yml index ffb5b37e6..57d0e65c6 100644 --- a/.github/workflows/mdns__host-tests.yml +++ b/.github/workflows/mdns__host-tests.yml @@ -9,7 +9,7 @@ on: jobs: host_test_mdns: - if: contains(github.event.pull_request.labels.*.name, 'mdns') + if: contains(github.event.pull_request.labels.*.name, 'mdns') || github.event_name == 'push' name: Host test runs-on: ubuntu-20.04 container: espressif/idf:latest @@ -33,7 +33,7 @@ jobs: cat ip.txt build_afl_host_test_mdns: - if: contains(github.event.pull_request.labels.*.name, 'mdns') + if: contains(github.event.pull_request.labels.*.name, 'mdns') || github.event_name == 'push' name: Build AFL host test strategy: matrix: diff --git a/.github/workflows/modem__build-host-tests.yml b/.github/workflows/modem__build-host-tests.yml index 64759ee2c..7993324d1 100644 --- a/.github/workflows/modem__build-host-tests.yml +++ b/.github/workflows/modem__build-host-tests.yml @@ -9,7 +9,7 @@ on: jobs: build_esp_modem: - if: contains(github.event.pull_request.labels.*.name, 'modem') + if: contains(github.event.pull_request.labels.*.name, 'modem') || github.event_name == 'push' name: Build strategy: matrix: @@ -47,7 +47,7 @@ jobs: python ./ci/build_apps.py components/esp_modem/examples/${{ matrix.example }} host_test_esp_modem: - if: contains(github.event.pull_request.labels.*.name, 'modem') + if: contains(github.event.pull_request.labels.*.name, 'modem') || github.event_name == 'push' name: Host Tests runs-on: ubuntu-20.04 container: espressif/idf:release-v4.3 @@ -84,7 +84,7 @@ jobs: files: esp-protocols/components/esp_modem/test/host_test/junit.xml host_test_gcov_esp_modem: - if: contains(github.event.pull_request.labels.*.name, 'modem') + if: contains(github.event.pull_request.labels.*.name, 'modem') || github.event_name == 'push' name: Run gcovr on esp modem host test runs-on: ubuntu-22.04 container: espressif/idf:release-v4.3 diff --git a/.github/workflows/modem__target-test.yml b/.github/workflows/modem__target-test.yml index cfb55e4c2..5a775c346 100644 --- a/.github/workflows/modem__target-test.yml +++ b/.github/workflows/modem__target-test.yml @@ -9,7 +9,7 @@ on: jobs: build_esp_modem_tests: - if: contains(github.event.pull_request.labels.*.name, 'modem') + if: contains(github.event.pull_request.labels.*.name, 'modem') || github.event_name == 'push' name: Build Target tests strategy: matrix: @@ -46,7 +46,9 @@ jobs: target_tests_esp_modem: # Skip running on forks since it won't have access to secrets - if: ${{ contains(github.event.pull_request.labels.*.name, 'modem') && (github.repository == 'espressif/esp-protocols') }} + if: | + github.repository == 'espressif/esp-protocols' && + ( contains(github.event.pull_request.labels.*.name, 'modem') || github.event_name == 'push' ) name: Run Target tests strategy: matrix: diff --git a/.github/workflows/mqtt_cxx__build.yml b/.github/workflows/mqtt_cxx__build.yml index 0f265f500..7f04f3dd5 100644 --- a/.github/workflows/mqtt_cxx__build.yml +++ b/.github/workflows/mqtt_cxx__build.yml @@ -9,7 +9,7 @@ on: jobs: build_esp_mqtt_cxx: - if: contains(github.event.pull_request.labels.*.name, 'mqtt') + if: contains(github.event.pull_request.labels.*.name, 'mqtt') || github.event_name == 'push' name: Build strategy: matrix: diff --git a/.github/workflows/websocket__build-target-test.yml b/.github/workflows/websocket__build-target-test.yml index 314fdbc90..f9947d75f 100644 --- a/.github/workflows/websocket__build-target-test.yml +++ b/.github/workflows/websocket__build-target-test.yml @@ -9,7 +9,7 @@ on: jobs: build_websocket: - if: contains(github.event.pull_request.labels.*.name, 'websocket') + if: contains(github.event.pull_request.labels.*.name, 'websocket') || github.event_name == 'push' name: Build strategy: matrix: @@ -56,7 +56,9 @@ jobs: run-target-websocket: # Skip running on forks since it won't have access to secrets - if: ${{ contains(github.event.pull_request.labels.*.name, 'websocket') && (github.repository == 'espressif/esp-protocols') }} + if: | + github.repository == 'espressif/esp-protocols' && + ( contains(github.event.pull_request.labels.*.name, 'websocket') || github.event_name == 'push' ) name: Target test needs: build_websocket strategy: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a0c3387c..e325e78a6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,3 +13,19 @@ Contributions in the form of pull requests, issue reports, and feature requests For quick merging, the contribution should be short, and concentrated on a single feature or topic. The larger the contribution is, the longer it would take to review it and merge it. Please follow the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) rule when writing commit messages. + +## Release process + +When releasing a new component version we have to: + +* Update the version number +* Update the changelog +* Create the version tag in this repository +* Deploy the component to component registry +* Update the documentation + +This process is not fully automated, the first three steps need to be performed manually by project maintainers running the `bump` command (from within this repository, rather than forks, to publish the release `tag`). Release procedure is as follows: +* Create a branch in this repository (not from fork) +* Run `cz bump [version]` (version number is optional, `cz` would automatically increment it if not present) +* Check the updated `CHANGELOG.md` +* Create and merge the branch to master From bf32e452d6a4819d2d1bf20e7608d80c3a857fe7 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Thu, 16 Mar 2023 09:26:07 +0100 Subject: [PATCH 4/4] fix(modem): warning in ap-2-pppos example when using lwip/napt Latest IDF added new napt API that support enabling NAPT based on netif pointer. Need to include "lwip/netif.h" --- components/esp_modem/examples/ap_to_pppos/main/ap_to_pppos.c | 1 + 1 file changed, 1 insertion(+) diff --git a/components/esp_modem/examples/ap_to_pppos/main/ap_to_pppos.c b/components/esp_modem/examples/ap_to_pppos/main/ap_to_pppos.c index 22ea76311..a6ac822d4 100644 --- a/components/esp_modem/examples/ap_to_pppos/main/ap_to_pppos.c +++ b/components/esp_modem/examples/ap_to_pppos/main/ap_to_pppos.c @@ -14,6 +14,7 @@ #include "esp_log.h" #include "esp_idf_version.h" #include "nvs_flash.h" +#include "lwip/netif.h" #include "lwip/lwip_napt.h" #include "freertos/FreeRTOS.h" #include "freertos/event_groups.h"