From c7ae67b7f0cfcc4b5b543786ca1b0a893a0c0fbd Mon Sep 17 00:00:00 2001 From: David Cermak Date: Tue, 15 Jun 2021 15:19:35 +0200 Subject: [PATCH 1/6] CI: Update QEMU image and test against IDF master(v4.4-dev) Also using test-apps instead of weekend tests. --- .gitlab-ci.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b427cc..9377ba0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,7 +90,7 @@ build_with_idf_v4: build_and_test_qemu: stage: build - image: ${CI_DOCKER_REGISTRY}/qemu:esp-develop-20191124 + image: ${CI_DOCKER_REGISTRY}/qemu:esp-develop-20210517-dda2caf0 tags: - build - shiny @@ -98,18 +98,19 @@ build_and_test_qemu: script: - cit_add_ssh_key "${GITLAB_KEY}" - git clone "${IDF_REPO}" - - $MQTT_PATH/ci/set_idf.sh release/v4.1 + - $MQTT_PATH/ci/set_idf.sh master - cd $IDF_PATH && tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" - $MQTT_PATH/ci/set_mqtt.sh $CI_COMMIT_SHA - # build publish stress test - - cd $IDF_PATH/examples/protocols/mqtt/publish_test && cat sdkconfig.qemu | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults && idf.py build - - source /opt/pyenv/activate && pyenv global 2.7.15 && python --version - # setup runner params + # build publish-connect stress test, setup test parameters + - cd tools/test_apps/protocols/mqtt/publish_connect_test && cat sdkconfig.qemu | $IDF_PATH/tools/ci/envsubst.py > sdkconfig.defaults && idf.py build + - export TEST_PATH=`pwd` && export MQTT_PUBLISH_TEST=1 + # setup python (TODO: cleanup, to be fixed when python2 support dropped) + - source /opt/pyenv/activate && pyenv global system && python3 --version + - python3 -m pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages" - - export TEST_CASE_PATH=$IDF_PATH/components/mqtt/weekend_test - - cd $IDF_PATH/tools/ci/python_packages/tiny_test_fw/bin # run test (with environment->qemu) - - python Runner.py $TEST_CASE_PATH -c $TEST_CASE_PATH/test_weekend_mqtt_qemu.yml -e $TEST_CASE_PATH/env.yml + - cd $IDF_PATH/tools/ci/python_packages/tiny_test_fw/bin + - python3 Runner.py $TEST_PATH -c $TEST_PATH/publish_connect_mqtt_qemu.yml -e $TEST_PATH/env.yml clang_tidy_check: stage: static_analysis From 626dc3645b613d58ef46e5e8d8601d264b2cef89 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Wed, 16 Jun 2021 07:55:41 +0200 Subject: [PATCH 2/6] CI: Add build test for IDF v4.3 --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9377ba0..1711f70 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,6 +72,11 @@ build_with_idf_v4: - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} && export EXTRA_CXXFLAGS=${EXTRA_CFLAGS} # build other examples - $MQTT_PATH/ci/build_examples.sh + # rebuild with IDFv4.3 + - $MQTT_PATH/ci/set_idf.sh release/v4.3 + - cd $IDF_PATH && tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" + - $MQTT_PATH/ci/set_mqtt.sh $CI_COMMIT_SHA + - $MQTT_PATH/ci/build_examples.sh # rebuild with IDFv4.2 - $MQTT_PATH/ci/set_idf.sh release/v4.2 - cd $IDF_PATH && tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" From 2e15c9a5fe98c341539b59f5d3980dfc197d27b5 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Wed, 16 Jun 2021 07:57:49 +0200 Subject: [PATCH 3/6] Fix IDF supported features: cert-bundle from v4.4 --- include/mqtt_supported_features.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/mqtt_supported_features.h b/include/mqtt_supported_features.h index eaad295..6e49182 100644 --- a/include/mqtt_supported_features.h +++ b/include/mqtt_supported_features.h @@ -57,6 +57,10 @@ // Features supported in 4.3 #define MQTT_SUPPORTED_FEATURE_DIGITAL_SIGNATURE #define MQTT_SUPPORTED_FEATURE_TRANSPORT_SOCK_ERRNO_REPORTING +#endif + +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(4, 4, 0) +// Features supported in 4.4 #define MQTT_SUPPORTED_FEATURE_CERTIFICATE_BUNDLE #endif From 1a94efe8b9e1b16c53eaa5c95e73e1c1337abbf5 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Sun, 20 Jun 2021 21:25:35 +0200 Subject: [PATCH 4/6] outbox: Cleanup all items when connection closes --- lib/include/mqtt_outbox.h | 1 + lib/mqtt_outbox.c | 6 +++++- mqtt_client.c | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/include/mqtt_outbox.h b/lib/include/mqtt_outbox.h index 21a10dd..7010e39 100644 --- a/lib/include/mqtt_outbox.h +++ b/lib/include/mqtt_outbox.h @@ -58,6 +58,7 @@ esp_err_t outbox_set_tick(outbox_handle_t outbox, int msg_id, outbox_tick_t tick int outbox_get_size(outbox_handle_t outbox); esp_err_t outbox_cleanup(outbox_handle_t outbox, int max_size); void outbox_destroy(outbox_handle_t outbox); +void outbox_delete_all_items(outbox_handle_t outbox); #ifdef __cplusplus } diff --git a/lib/mqtt_outbox.c b/lib/mqtt_outbox.c index 77ef3ec..e6fc12b 100644 --- a/lib/mqtt_outbox.c +++ b/lib/mqtt_outbox.c @@ -224,7 +224,7 @@ esp_err_t outbox_cleanup(outbox_handle_t outbox, int max_size) return ESP_OK; } -void outbox_destroy(outbox_handle_t outbox) +void outbox_delete_all_items(outbox_handle_t outbox) { outbox_item_handle_t item, tmp; STAILQ_FOREACH_SAFE(item, outbox, next, tmp) { @@ -232,6 +232,10 @@ void outbox_destroy(outbox_handle_t outbox) free(item->buffer); free(item); } +} +void outbox_destroy(outbox_handle_t outbox) +{ + outbox_delete_all_items(outbox); free(outbox); } diff --git a/mqtt_client.c b/mqtt_client.c index a71b1fb..4ec8f14 100644 --- a/mqtt_client.c +++ b/mqtt_client.c @@ -1494,6 +1494,7 @@ static void esp_mqtt_task(void *pv) } esp_transport_close(client->transport); + outbox_delete_all_items(client->outbox); xEventGroupSetBits(client->status_bits, STOPPED_BIT); vTaskDelete(NULL); From 90f0bf8dd4f41770a4bed27a11a0bb39cf2527cb Mon Sep 17 00:00:00 2001 From: David Cermak Date: Thu, 24 Jun 2021 07:40:43 +0200 Subject: [PATCH 5/6] Updated docker images to work with latest IDFv4.4 --- .gitlab-ci.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1711f70..e9f741c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,7 +47,7 @@ build_with_idf_v3: build_with_idf_v4: stage: build - image: ${CI_DOCKER_REGISTRY}/esp32-ci-env + image: ${CI_DOCKER_REGISTRY}/esp-env:v4.4-1 tags: - build dependencies: [] @@ -56,9 +56,12 @@ build_with_idf_v4: paths: - tidybuild/* expire_in: 1 day + variables: + PYTHON_VER: 3.6.13 script: - cit_add_ssh_key "${GITLAB_KEY}" - git clone "${IDF_REPO}" + - source /opt/pyenv/activate && pyenv global $PYTHON_VER - $MQTT_PATH/ci/set_idf.sh master - cd $IDF_PATH && tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" - $MQTT_PATH/ci/set_mqtt.sh $CI_COMMIT_SHA @@ -100,22 +103,23 @@ build_and_test_qemu: - build - shiny dependencies: [] + variables: + PYTHON_VER: 3.6.13 script: - cit_add_ssh_key "${GITLAB_KEY}" - git clone "${IDF_REPO}" + # switch to IDF and setup the right tools - $MQTT_PATH/ci/set_idf.sh master + - source /opt/pyenv/activate && pyenv global $PYTHON_VER - cd $IDF_PATH && tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" - $MQTT_PATH/ci/set_mqtt.sh $CI_COMMIT_SHA # build publish-connect stress test, setup test parameters - cd tools/test_apps/protocols/mqtt/publish_connect_test && cat sdkconfig.qemu | $IDF_PATH/tools/ci/envsubst.py > sdkconfig.defaults && idf.py build - export TEST_PATH=`pwd` && export MQTT_PUBLISH_TEST=1 - # setup python (TODO: cleanup, to be fixed when python2 support dropped) - - source /opt/pyenv/activate && pyenv global system && python3 --version - - python3 -m pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages" # run test (with environment->qemu) - cd $IDF_PATH/tools/ci/python_packages/tiny_test_fw/bin - - python3 Runner.py $TEST_PATH -c $TEST_PATH/publish_connect_mqtt_qemu.yml -e $TEST_PATH/env.yml + - python Runner.py $TEST_PATH -c $TEST_PATH/publish_connect_mqtt_qemu.yml -e $TEST_PATH/env.yml clang_tidy_check: stage: static_analysis From a7e5accc7ef3e69129d95e8bc0ddeadb8ac13bba Mon Sep 17 00:00:00 2001 From: David Cermak Date: Thu, 24 Jun 2021 08:32:51 +0200 Subject: [PATCH 6/6] CI: Temporariy update QEMU image with python 3.7.0 This will be removed once we updated QEMU image, based on the new CI environment docker image. --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e9f741c..503b621 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -104,14 +104,16 @@ build_and_test_qemu: - shiny dependencies: [] variables: - PYTHON_VER: 3.6.13 + PYTHON_VER: 3.7.0 script: - cit_add_ssh_key "${GITLAB_KEY}" - git clone "${IDF_REPO}" # switch to IDF and setup the right tools - $MQTT_PATH/ci/set_idf.sh master - - source /opt/pyenv/activate && pyenv global $PYTHON_VER + - source /opt/pyenv/activate && pyenv install $PYTHON_VER && pyenv global $PYTHON_VER - cd $IDF_PATH && tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" + - ./install.sh && source export.sh + - python -m pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt && python -m pip install paho-mqtt - $MQTT_PATH/ci/set_mqtt.sh $CI_COMMIT_SHA # build publish-connect stress test, setup test parameters - cd tools/test_apps/protocols/mqtt/publish_connect_test && cat sdkconfig.qemu | $IDF_PATH/tools/ci/envsubst.py > sdkconfig.defaults && idf.py build