Merge pull request #466 from gabsuren/websocket_bump_1.2.2

bump(websocket): 1.2.1 -> 1.2.2

.github/workflows/console_cmd_ifconfig__build.yml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         idf_ver: ["latest", "release-v5.0"]
         idf_target: ["esp32"]
-        test: [ { app: example, path: "components/console_cmd_ifconfig/examples" }]
+        test: [ { app: ifconfig-basic, path: "components/console_cmd_ifconfig/examples"}]
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
@@ -29,4 +29,4 @@ jobs:
         run: |
           ${IDF_PATH}/install.sh --enable-pytest
           . ${IDF_PATH}/export.sh
-          python $IDF_PATH/tools/ci/ci_build_apps.py . --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app
+          python ../../../ci/build_apps.py ./${{ matrix.test.app }} --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app

.github/workflows/console_cmd_ping__build.yml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         idf_ver: ["latest", "release-v5.0"]
         idf_target: ["esp32"]
-        test: [ { app: example, path: "components/console_cmd_ping/examples" }]
+        test: [ { app: ping-basic, path: "components/console_cmd_ping/examples" }]
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
@@ -29,4 +29,4 @@ jobs:
         run: |
           ${IDF_PATH}/install.sh --enable-pytest
           . ${IDF_PATH}/export.sh
-          python $IDF_PATH/tools/ci/ci_build_apps.py . --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app
+          python ../../../ci/build_apps.py ./${{ matrix.test.app }} --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app

.github/workflows/console_cmd_wifi__build.yml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         idf_ver: ["latest", "release-v5.0"]
         idf_target: ["esp32"]
-        test: [ { app: example, path: "components/console_cmd_wifi/examples" }]
+        test: [ { app: wifi-basic, path: "components/console_cmd_wifi/examples" }]
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
@@ -29,4 +29,4 @@ jobs:
         run: |
           ${IDF_PATH}/install.sh --enable-pytest
           . ${IDF_PATH}/export.sh
-          python $IDF_PATH/tools/ci/ci_build_apps.py . --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app
+          python ../../../ci/build_apps.py ./${{ matrix.test.app }} --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app

.github/workflows/console_simple_init__build.yml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         idf_ver: ["latest", "release-v5.0"]
         idf_target: ["esp32"]
-        test: [ { app: example, path: "components/console_simple_init/examples" }]
+        test: [ { app: console_basic, path: "components/console_simple_init/examples" }]
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
@@ -29,4 +29,4 @@ jobs:
         run: |
           ${IDF_PATH}/install.sh --enable-pytest
           . ${IDF_PATH}/export.sh
-          python $IDF_PATH/tools/ci/ci_build_apps.py . --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app
+          python ../../../ci/build_apps.py ./${{ matrix.test.app }} --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app

.github/workflows/modem__build-host-tests.yml

@@ -52,7 +52,7 @@ jobs:
     strategy:
       matrix:
         idf_ver: ["release-v5.0", "release-v5.1", "latest"]
-        test: ["target", "target_ota"]
+        test: ["target", "target_ota", "target_iperf"]
 
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}

.github/workflows/mqtt_cxx__build.yml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         idf_ver: ["latest", "release-v5.0"]
         idf_target: ["esp32"]
-        test: [ { app: example, path: "components/esp_mqtt_cxx/examples" }]
+        test: [ { app: mqtt-basic, path: "components/esp_mqtt_cxx/examples" }]
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
@@ -29,4 +29,4 @@ jobs:
         run: |
           ${IDF_PATH}/install.sh --enable-pytest
           . ${IDF_PATH}/export.sh
-          python $IDF_PATH/tools/ci/ci_build_apps.py . --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app
+          python ../../../ci/build_apps.py ./${{ matrix.test.app }} --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app

.github/workflows/publish-docs-component.yml

@@ -48,7 +48,7 @@ jobs:
         run: |
             sudo apt-get update
             sudo apt-get -y install doxygen clang python3-pip
-            python -m pip install breathe recommonmark esp-docs==1.4.1
+            python -m pip install -r docs/requirements.txt
             for comp in  `ls components`; do
               if [[ -d $GITHUB_WORKSPACE/docs/${comp} ]]; then
                 cd $GITHUB_WORKSPACE/docs/${comp}

ci/build_apps.py

@@ -10,6 +10,8 @@ import sys
 
 from idf_build_apps import build_apps, find_apps, setup_logging
 from idf_build_apps.constants import SUPPORTED_TARGETS
+from packaging import version
+from pkg_resources import get_distribution
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
@@ -17,6 +19,12 @@ if __name__ == '__main__':
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
     parser.add_argument('paths', nargs='+', help='Paths to the apps to build.')
+    parser.add_argument(
+        '-v',
+        '--verbose',
+        action='count',
+        help='Increase the LOGGER level of the script. Can be specified multiple times.',
+    )
     parser.add_argument(
         '-t',
         '--target',
@@ -28,6 +36,8 @@ if __name__ == '__main__':
     parser.add_argument('-d', '--delete', action='store_true', help='Delete build artifacts')
     parser.add_argument('-c', '--recursive', action='store_true', help='Build recursively')
     parser.add_argument('-l', '--linux', action='store_true', help='Include linux build (dont check warnings)')
+    parser.add_argument('--preserve-all', action='store_true', help='Preserve the binaries for all apps when specified.')
+    parser.add_argument('--pytest-apps', action='store_true', help='Only build apps required by pytest scripts.')
     args = parser.parse_args()
 
     IDF_PATH = os.environ['IDF_PATH']
@@ -41,20 +51,36 @@ if __name__ == '__main__':
         SUPPORTED_TARGETS.append('linux')
         ignore_warning = 'warning: '  # Ignore all common warnings on linux builds
     setup_logging(2)
-    apps = find_apps(
-        args.paths,
-        recursive=args.recursive,
-        target=args.target,
-        build_dir='build_@t_@w',
-        config_rules_str=args.rules,
-        build_log_path='build_log.txt',
-        size_json_path='size.json' if not args.linux else None,
-        check_warnings=True,
-        preserve=not args.delete,
-        manifest_files=args.manifests,
-        default_build_targets=SUPPORTED_TARGETS,
-        manifest_rootpath='.',
-    )
+    if version.parse(get_distribution('idf_build_apps').version) >= version.parse('2.0.0'):
+        apps = find_apps(
+            args.paths,
+            recursive=args.recursive,
+            target=args.target,
+            build_dir='build_@t_@w',
+            config_rules_str=args.rules,
+            build_log_filename='build_log.txt',
+            size_json_filename='size.json' if not args.linux else None,
+            check_warnings=True,
+            preserve=not args.delete,
+            manifest_files=args.manifests,
+            default_build_targets=SUPPORTED_TARGETS,
+            manifest_rootpath='.',
+        )
+    else:
+        apps = find_apps(
+            args.paths,
+            recursive=args.recursive,
+            target=args.target,
+            build_dir='build_@t_@w',
+            config_rules_str=args.rules,
+            build_log_path='build_log.txt',
+            size_json_path='size.json' if not args.linux else None,
+            check_warnings=True,
+            preserve=not args.delete,
+            manifest_files=args.manifests,
+            default_build_targets=SUPPORTED_TARGETS,
+            manifest_rootpath='.',
+        )
 
     for app in apps:
         print(app)

components/esp_modem/examples/modem_console/main/my_module_dce.hpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Unlicense OR CC0-1.0
  */
@@ -17,26 +17,20 @@
 #include "cxx_include/esp_modem_dce_module.hpp"
 
 /**
- * @brief Definition of a custom modem which inherits from the GenericModule, uses all its methods
- * and could override any of them. Here, for demonstration purposes only, we redefine just `get_module_name()`
+ * @brief Definition of a custom DCE uses GenericModule and all its methods
+ * but could override command processing. Here, for demonstration purposes only,
+ * we "inject" URC handler to the actual command processing.
+ * This is possible since we inherit from `CommandableIf` and redefine `command()` method.
+ * Then we're able to use declare all common methods from the command library
+ * to be processed using "our" `command()` method (with custom URC handler).
  */
-class MyShinyModem: public esp_modem::GenericModule {
-    using GenericModule::GenericModule;
-public:
-    esp_modem::command_result get_module_name(std::string &name) override
-    {
-        name = "Custom Shiny Module";
-        return esp_modem::command_result::OK;
-    }
-};
-
 namespace Shiny {
 
 using namespace esp_modem;
 
-class DCE : public esp_modem::DCE_T<MyShinyModem>, public CommandableIf {
+class DCE : public esp_modem::DCE_T<GenericModule>, public CommandableIf {
 public:
-    using DCE_T<MyShinyModem>::DCE_T;
+    using DCE_T<GenericModule>::DCE_T;
 
     command_result
     command(const std::string &cmd, got_line_cb got_line, uint32_t time_ms) override
@@ -97,7 +91,7 @@ public:
                                        std::shared_ptr<esp_modem::DTE> dte,
                                        esp_netif_t *netif)
     {
-        return build_generic_DCE<MyShinyModem, DCE, std::unique_ptr<DCE>>(config, std::move(dte), netif);
+        return build_generic_DCE<GenericModule, DCE, std::unique_ptr<DCE>>(config, std::move(dte), netif);
     }
 
 };

components/esp_modem/include/cxx_include/esp_modem_command_library.hpp

@@ -54,7 +54,7 @@ command_result power_down_sim76xx(CommandableIf *t);
 command_result power_down_sim70xx(CommandableIf *t);
 command_result set_network_bands_sim76xx(CommandableIf *t, const std::string &mode, const int *bands, int size);
 command_result power_down_sim8xx(CommandableIf *t);
-command_result set_data_mode_sim8xx(CommandableIf *t);
+command_result set_data_mode_alt(CommandableIf *t);
 command_result set_pdp_context(CommandableIf *t, PdpContext &pdp, uint32_t timeout_ms);
 
 /**

components/esp_modem/include/cxx_include/esp_modem_command_library_utils.hpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -25,18 +25,16 @@ command_result generic_command(CommandableIf *t, const std::string &command,
  * @brief Utility command to send command and return reply (after DCE says OK)
  * @param t Anything that is "command-able"
  * @param command Command to issue
- * @param output String to return
- * @param timeout_ms
+ * @param output String to return (could be either std::string& or std::string_view&)
  * @param timeout_ms Command timeout in ms
  * @return Generic command return type (OK, FAIL, TIMEOUT)
  */
-command_result generic_get_string(CommandableIf *t, const std::string &command, std::string &output, uint32_t timeout_ms = 500);
+template <typename T> command_result generic_get_string(CommandableIf *t, const std::string &command, T &output, uint32_t timeout_ms = 500);
 
 /**
  * @brief Generic command that passes on "OK" and fails on "ERROR"
  * @param t Anything that is "command-able"
  * @param command Command to issue
- * @param timeout
  * @param timeout_ms Command timeout in ms
  * @return Generic command return type (OK, FAIL, TIMEOUT)
  */

components/esp_modem/include/cxx_include/esp_modem_dce_module.hpp

@@ -144,6 +144,8 @@ class SIM7070: public GenericModule {
     using GenericModule::GenericModule;
 public:
     command_result power_down() override;
+    command_result set_data_mode() override;
+
 };
 
 /**
@@ -162,7 +164,6 @@ class SIM800: public GenericModule {
     using GenericModule::GenericModule;
 public:
     command_result power_down() override;
-    command_result set_data_mode() override;
 };
 
 /**

components/esp_modem/include/generate/esp_modem_command_declare.inc

@@ -44,6 +44,18 @@ ESP_MODEM_DECLARE_DCE_COMMAND(store_profile, command_result, 0) \
  */\
 ESP_MODEM_DECLARE_DCE_COMMAND(set_pin, command_result, 1, STRING_IN(p1, pin)) \
     \
+/**
+ * @brief Execute the supplied AT command in raw mode (doesn't append '\r' to command, returns everything)
+ * @param[in] cmd String command that's send to DTE
+ * @param[out] out Raw output from DTE
+ * @param[in] pass Pattern in response for the API to return OK
+ * @param[in] fail Pattern in response for the API to return FAIL
+ * @param[in] cmd String command that's send to DTE
+ * @param[in] timeout AT command timeout in milliseconds
+ * @return OK, FAIL or TIMEOUT
+ */\
+ESP_MODEM_DECLARE_DCE_COMMAND(at_raw, command_result, 5, STRING_IN(p1, cmd), STRING_OUT(p2, out), STRING_IN(p3, pass), STRING_IN(p4, fail), INT_IN(p5, timeout)) \
+    \
 /**
  * @brief Execute the supplied AT command
  * @param[in] cmd AT command

components/esp_modem/src/esp_modem_c_api.cpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -17,7 +17,7 @@
 #include "esp_private/c_api_wrapper.hpp"
 
 #ifndef ESP_MODEM_C_API_STR_MAX
-#define ESP_MODEM_C_API_STR_MAX 64
+#define ESP_MODEM_C_API_STR_MAX 128
 #endif
 
 #ifndef HAVE_STRLCPY
@@ -206,6 +206,20 @@ extern "C" esp_err_t esp_modem_get_imsi(esp_modem_dce_t *dce_wrap, char *p_imsi)
     return ret;
 }
 
+extern "C" esp_err_t esp_modem_at_raw(esp_modem_dce_t *dce_wrap, const char *cmd, char *p_out, const char *pass, const char *fail, int timeout)
+{
+    if (dce_wrap == nullptr || dce_wrap->dce == nullptr) {
+        return ESP_ERR_INVALID_ARG;
+    }
+    std::string out;
+    auto ret = command_response_to_esp_err(dce_wrap->dce->at_raw(cmd, out, pass, fail, timeout));
+    if ((p_out != NULL) && (!out.empty())) {
+        strlcpy(p_out, out.c_str(), ESP_MODEM_C_API_STR_MAX);
+    }
+    return ret;
+}
+
+
 extern "C" esp_err_t esp_modem_set_flow_control(esp_modem_dce_t *dce_wrap, int dce_flow, int dte_flow)
 {
     if (dce_wrap == nullptr || dce_wrap->dce == nullptr) {

components/esp_modem/src/esp_modem_command_library.cpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -51,7 +51,35 @@ command_result generic_command(CommandableIf *t, const std::string &command,
     return generic_command(t, command, pass, fail, timeout_ms);
 }
 
-static command_result generic_get_string(CommandableIf *t, const std::string &command, std::string_view &output, uint32_t timeout_ms = 500)
+/*
+ * Purpose of this namespace is to provide different means of assigning the result to a string-like parameter.
+ * By default we assign strings, which comes with an allocation. Alternatively we could take `std::span`
+ * with user's buffer and directly copy the result, thus avoiding allocations (this is not used as of now)
+ */
+namespace str_copy {
+
+bool set(std::string &dest, std::string_view &src)
+{
+    dest = src;
+    return true;
+}
+
+/* This is an example of using std::span output in generic_get_string()
+bool set(std::span<char> &dest, std::string_view &src)
+{
+    if (dest.size() >= src.size()) {
+        std::copy(src.begin(), src.end(), dest.data());
+        dest = dest.subspan(0, src.size());
+        return true;
+    }
+    ESP_LOGE(TAG, "Cannot set result of size %d (to span of size %d)", dest.size(), src.size());
+    return false;
+}
+*/
+
+} // str_copy
+
+template <typename T> command_result generic_get_string(CommandableIf *t, const std::string &command, T &output, uint32_t timeout_ms)
 {
     ESP_LOGV(TAG, "%s", __func__ );
     return t->command(command, [&](uint8_t *data, size_t len) {
@@ -70,7 +98,9 @@ static command_result generic_get_string(CommandableIf *t, const std::string &co
             } else if (token.find("ERROR") != std::string::npos) {
                 return command_result::FAIL;
             } else if (token.size() > 2) {
-                output = token;
+                if (!str_copy::set(output, token)) {
+                    return command_result::FAIL;
+                }
             }
             response = response.substr(pos + 1);
         }
@@ -78,18 +108,6 @@ static command_result generic_get_string(CommandableIf *t, const std::string &co
     }, timeout_ms);
 }
 
-command_result generic_get_string(CommandableIf *t, const std::string &command, std::string &output, uint32_t timeout_ms)
-{
-    ESP_LOGV(TAG, "%s", __func__ );
-    std::string_view out;
-    auto ret = generic_get_string(t, command, out, timeout_ms);
-    if (ret == command_result::OK) {
-        output = out;
-    }
-    return ret;
-}
-
-
 command_result generic_command_common(CommandableIf *t, const std::string &command, uint32_t timeout_ms)
 {
     ESP_LOGV(TAG, "%s", __func__ );
@@ -153,7 +171,7 @@ command_result hang_up(CommandableIf *t)
 command_result get_battery_status(CommandableIf *t, int &voltage, int &bcs, int &bcl)
 {
     ESP_LOGV(TAG, "%s", __func__ );
-    std::string_view out;
+    std::string out;
     auto ret = generic_get_string(t, "AT+CBC\r", out);
     if (ret != command_result::OK) {
         return ret;
@@ -189,7 +207,7 @@ command_result get_battery_status(CommandableIf *t, int &voltage, int &bcs, int
 command_result get_battery_status_sim7xxx(CommandableIf *t, int &voltage, int &bcs, int &bcl)
 {
     ESP_LOGV(TAG, "%s", __func__ );
-    std::string_view out;
+    std::string out;
     auto ret = generic_get_string(t, "AT+CBC\r", out);
     if (ret != command_result::OK) {
         return ret;
@@ -224,7 +242,7 @@ command_result set_flow_control(CommandableIf *t, int dce_flow, int dte_flow)
 command_result get_operator_name(CommandableIf *t, std::string &operator_name, int &act)
 {
     ESP_LOGV(TAG, "%s", __func__ );
-    std::string_view out;
+    std::string out;
     auto ret = generic_get_string(t, "AT+COPS?\r", out, 75000);
     if (ret != command_result::OK) {
         return ret;
@@ -277,13 +295,13 @@ command_result set_pdp_context(CommandableIf *t, PdpContext &pdp)
 command_result set_data_mode(CommandableIf *t)
 {
     ESP_LOGV(TAG, "%s", __func__ );
-    return generic_command(t, "ATD*99##\r", "CONNECT", "ERROR", 5000);
+    return generic_command(t, "ATD*99#\r", "CONNECT", "ERROR", 5000);
 }
 
-command_result set_data_mode_sim8xx(CommandableIf *t)
+command_result set_data_mode_alt(CommandableIf *t)
 {
     ESP_LOGV(TAG, "%s", __func__ );
-    return generic_command(t, "ATD*99#\r", "CONNECT", "ERROR", 5000);
+    return generic_command(t, "ATD*99##\r", "CONNECT", "ERROR", 5000);
 }
 
 command_result resume_data_mode(CommandableIf *t)
@@ -361,7 +379,7 @@ command_result set_cmux(CommandableIf *t)
 command_result read_pin(CommandableIf *t, bool &pin_ok)
 {
     ESP_LOGV(TAG, "%s", __func__ );
-    std::string_view out;
+    std::string out;
     auto ret = generic_get_string(t, "AT+CPIN?\r", out);
     if (ret != command_result::OK) {
         return ret;
@@ -394,10 +412,26 @@ command_result at(CommandableIf *t, const std::string &cmd, std::string &out, in
     return generic_get_string(t, at_command, out, timeout);
 }
 
+command_result at_raw(CommandableIf *t, const std::string &cmd, std::string &out, const std::string &pass, const std::string &fail, int timeout = 500)
+{
+    ESP_LOGV(TAG, "%s", __func__ );
+    return t->command(cmd, [&](uint8_t *data, size_t len) {
+        out.assign(reinterpret_cast<char *>(data), len);
+
+        if (out.find(pass) != std::string::npos) {
+            return command_result::OK;
+        } else if (out.find(fail) != std::string::npos) {
+            return command_result::FAIL;
+        }
+
+        return command_result::TIMEOUT;
+    }, timeout);
+}
+
 command_result get_signal_quality(CommandableIf *t, int &rssi, int &ber)
 {
     ESP_LOGV(TAG, "%s", __func__ );
-    std::string_view out;
+    std::string out;
     auto ret = generic_get_string(t, "AT+CSQ\r", out);
     if (ret != command_result::OK) {
         return ret;
@@ -435,7 +469,7 @@ command_result set_network_attachment_state(CommandableIf *t, int state)
 command_result get_network_attachment_state(CommandableIf *t, int &state)
 {
     ESP_LOGV(TAG, "%s", __func__ );
-    std::string_view out;
+    std::string out;
     auto ret = generic_get_string(t, "AT+CGATT?\r", out);
     if (ret != command_result::OK) {
         return ret;
@@ -462,7 +496,7 @@ command_result set_radio_state(CommandableIf *t, int state)
 command_result get_radio_state(CommandableIf *t, int &state)
 {
     ESP_LOGV(TAG, "%s", __func__ );
-    std::string_view out;
+    std::string out;
     auto ret = generic_get_string(t, "AT+CFUN?\r", out);
     if (ret != command_result::OK) {
         return ret;
@@ -527,7 +561,7 @@ command_result set_network_bands_sim76xx(CommandableIf *t, const std::string &mo
 command_result get_network_system_mode(CommandableIf *t, int &mode)
 {
     ESP_LOGV(TAG, "%s", __func__ );
-    std::string_view out;
+    std::string out;
     auto ret = generic_get_string(t, "AT+CNSMOD?\r", out);
     if (ret != command_result::OK) {
         return ret;
@@ -555,7 +589,7 @@ command_result set_gnss_power_mode(CommandableIf *t, int mode)
 command_result get_gnss_power_mode(CommandableIf *t, int &mode)
 {
     ESP_LOGV(TAG, "%s", __func__ );
-    std::string_view out;
+    std::string out;
     auto ret = generic_get_string(t, "AT+CGNSPWR?\r", out);
     if (ret != command_result::OK) {
         return ret;

components/esp_modem/src/esp_modem_modules.cpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -67,6 +67,11 @@ command_result SIM7070::power_down()
     return dce_commands::power_down_sim70xx(dte.get());
 }
 
+command_result SIM7070::set_data_mode()
+{
+    return dce_commands::set_data_mode_alt(dte.get());
+}
+
 command_result SIM7000::power_down()
 {
     return dce_commands::power_down_sim70xx(dte.get());
@@ -77,11 +82,6 @@ command_result SIM800::power_down()
     return dce_commands::power_down_sim8xx(dte.get());
 }
 
-command_result SIM800::set_data_mode()
-{
-    return dce_commands::set_data_mode_sim8xx(dte.get());
-}
-
 command_result BG96::set_pdp_context(esp_modem::PdpContext &pdp)
 {
     return dce_commands::set_pdp_context(dte.get(), pdp, 300);

components/esp_modem/test/host_test/main/LoopbackTerm.cpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Unlicense OR CC0-1.0
  */
@@ -36,7 +36,7 @@ int LoopbackTerm::write(uint8_t *data, size_t len)
         } else if (command == "ATO\r") {
             response = "ERROR\r\n";
         } else if (command.find("ATD") != std::string::npos) {
-            response = "CONNECT\r\n";
+            response = "CONNECT\n";
         } else if (command.find("AT+CSQ\r") != std::string::npos) {
             response = "+CSQ: 123,456\n\r\nOK\r\n";
         } else if (command.find("AT+CGMM\r") != std::string::npos) {
@@ -74,6 +74,9 @@ int LoopbackTerm::write(uint8_t *data, size_t len)
     }
     if (len > 2 && data[0] == 0xf9) { // Simple CMUX responder
         // turn the request into a reply -> implements CMUX loopback
+        // Note: This simple CMUX responder only updates CMUX headers and replaces payload.
+        // It means that all responses (that we test) must be shorter or equal to the requests
+        // For example ATD (dial command): sizeof("ATD*99#") >= sizeof("CONNECT");
         if (data[2] == 0x3f || data[2] == 0x53) {  // SABM command
             data[2] = 0x73;
         } else if (data[2] == 0xef) { // Generic request

components/esp_modem/test/target_iperf/CMakeLists.txt

@@ -0,0 +1,10 @@
+# The following five lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+set(EXTRA_COMPONENT_DIRS    $ENV{IDF_PATH}/examples/system/console/advanced/components
+                            $ENV{IDF_PATH}/examples/common_components/iperf
+                            "../..")
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(pppd_test)

components/esp_modem/test/target_iperf/README.md

@@ -0,0 +1,48 @@
+# Target test for measuring TCP/UDP network performance
+
+## Overview
+
+The aim of this test is to run `iperf` application to measure network throughput of the esp_modem library.
+
+### Configure PPP server
+
+This test uses a network DCE device, which only needs a PPP server. To run the PPP server, use this command:
+```
+sudo pppd /dev/ttyUSB1 115200 192.168.11.1:192.168.11.2 ms-dns 8.8.8.8 modem local noauth debug nocrtscts nodetach +ipv6
+```
+
+### Running using pytest
+
+For checking the performance, you only need to execute the pytest, which will measure the network throughput automatically and report the resultant values. For running the pytest, you need to:
+* install IDF pytest packages by running: `./install.sh --enable-pytest`
+* add IDF internal packages to python path: `export PYTHONPATH=${DIF_PATH}/tools/ci/python_packages/`
+* run the pytest as **superuser**
+
+It's useful to note that when running the test multiple times, you can use `pytest --skip-autoflash y` so the pytest wouldn't have to always reprogram the DUT.
+
+### Performance summary
+
+Here's an example of the resultant summary logged by the pytest
+```
+2023-11-29 18:28:25 INFO [Performance][tcp_tx_throughput]: 0.75 Mbps
+2023-11-29 18:28:25 INFO [Performance][tcp_rx_throughput]: 0.70 Mbps
+2023-11-29 18:28:25 INFO [Performance][udp_tx_throughput]: 0.73 Mbps
+2023-11-29 18:28:25 INFO [Performance][udp_rx_throughput]: 0.70 Mbps
+```
+
+### Running the iperf manually
+
+Execute `idf.py flash monitor` in one terminal and after connecting to the PPP server (after getting an IP address), you can use standard `iperf` commands.
+In another terminal, you need to execute the iperf counterpart.
+For example running for checking UDP performance, and running server on ESP32, please run:
+* iperf -u -s -i 3 (in ESP32 terminal)
+* iperf -u -c SERVER_IP -t 60 -i 3 (on the host side)
+
+Note that command `pppd info` will print actual IP addresses:
+```
+iperf> pppd info
+ppp:
+  IP: 192.168.11.2
+  MASK: 255.255.255.255
+  GW: 192.168.11.3
+```

components/esp_modem/test/target_iperf/main/CMakeLists.txt

@@ -0,0 +1,3 @@
+idf_component_register(SRCS "NetworkDCE.cpp"
+                            "cmd_pppclient.c"
+                            "pppd_iperf_main.c")

components/esp_modem/test/target_iperf/main/NetworkDCE.cpp

@@ -0,0 +1,96 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#include "cxx_include/esp_modem_dte.hpp"
+#include "esp_modem_config.h"
+#include "cxx_include/esp_modem_api.hpp"
+#include "cxx_include/esp_modem_dce_factory.hpp"
+#include <memory>
+#include <utility>
+
+using namespace esp_modem;
+using namespace esp_modem::dce_factory;
+
+class NetModule;
+typedef DCE_T<NetModule> NetDCE;
+
+/**
+ * @brief Custom factory which can build and create a DCE using a custom module
+ */
+class NetDCE_Factory: public Factory {
+public:
+    template <typename T, typename ...Args>
+    static DCE_T<T> *create(const config *cfg, Args &&... args)
+    {
+        return build_generic_DCE<T>(cfg, std::forward<Args>(args)...);
+    }
+};
+
+/**
+ * @brief This is a null-module, doesn't define any AT commands, just passes everything to pppd
+ */
+class NetModule: public ModuleIf {
+public:
+    explicit NetModule(std::shared_ptr<DTE> dte, const esp_modem_dce_config *cfg):
+        dte(std::move(dte)) {}
+
+    bool setup_data_mode() override
+    {
+        return true;
+    }
+
+    bool set_mode(modem_mode mode) override
+    {
+        return true;
+    }
+
+    static esp_err_t init(esp_netif_t *netif)
+    {
+        // configure
+        esp_modem_dte_config_t dte_config = ESP_MODEM_DTE_DEFAULT_CONFIG();
+        dte_config.uart_config.baud_rate = 921600;  // check also 460800
+        esp_modem_dce_config dce_config = ESP_MODEM_DCE_DEFAULT_CONFIG("");
+
+        // create DTE and minimal network DCE
+        auto uart_dte = create_uart_dte(&dte_config);
+        dce = NetDCE_Factory::create<NetModule>(&dce_config, uart_dte, netif);
+        return dce == nullptr ? ESP_FAIL : ESP_OK;
+    }
+
+    static void deinit()
+    {
+        delete dce;
+    }
+    static void start()
+    {
+        dce->set_data();
+    }
+    static void stop()
+    {
+        dce->exit_data();
+    }
+
+private:
+    static NetDCE *dce;
+    std::shared_ptr<DTE> dte;
+};
+
+NetDCE *NetModule::dce = nullptr;
+
+extern "C" esp_err_t modem_init_network(esp_netif_t *netif)
+{
+    return NetModule::init(netif);
+}
+
+extern "C" esp_err_t modem_start_network()
+{
+    NetModule::start();
+    return ESP_OK;
+}
+
+extern "C" void modem_stop_network()
+{
+    NetModule::stop();
+}

components/esp_modem/test/target_iperf/main/cmd_pppclient.c

@@ -0,0 +1,289 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include "freertos/FreeRTOS.h"
+#include "freertos/event_groups.h"
+#include "sys/socket.h" // for INADDR_ANY
+#include "esp_netif.h"
+#include "esp_log.h"
+#include "esp_system.h"
+#include "esp_event.h"
+#include "esp_log.h"
+#include "esp_netif.h"
+#include "esp_netif_ppp.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/event_groups.h"
+
+#include "esp_console.h"
+#include "esp_event.h"
+#include "esp_bit_defs.h"
+#include "argtable3/argtable3.h"
+#include "iperf.h"
+#include "sdkconfig.h"
+#include "network_dce.h"
+
+static const char *TAG = "pppd_test";
+static EventGroupHandle_t event_group = NULL;
+static esp_netif_t *s_ppp_netif;
+static const int GOTIP_BIT = BIT0;
+
+static void on_modem_event(void *arg, esp_event_base_t event_base,
+                           int32_t event_id, void *event_data)
+{
+    if (event_base == IP_EVENT) {
+        ESP_LOGD(TAG, "IP event! %" PRIu32, event_id);
+        if (event_id == IP_EVENT_PPP_GOT_IP) {
+            esp_netif_dns_info_t dns_info;
+
+
+            ip_event_got_ip_t *event = (ip_event_got_ip_t *)event_data;
+            esp_netif_t *netif = event->esp_netif;
+
+            ESP_LOGI(TAG, "Modem Connect to PPP Server");
+            ESP_LOGI(TAG, "~~~~~~~~~~~~~~");
+            ESP_LOGI(TAG, "IP          : " IPSTR, IP2STR(&event->ip_info.ip));
+            ESP_LOGI(TAG, "Netmask     : " IPSTR, IP2STR(&event->ip_info.netmask));
+            ESP_LOGI(TAG, "Gateway     : " IPSTR, IP2STR(&event->ip_info.gw));
+            esp_netif_get_dns_info(netif, ESP_NETIF_DNS_MAIN, &dns_info);
+            ESP_LOGI(TAG, "Name Server1: " IPSTR, IP2STR(&dns_info.ip.u_addr.ip4));
+            esp_netif_get_dns_info(netif, ESP_NETIF_DNS_BACKUP, &dns_info);
+            ESP_LOGI(TAG, "Name Server2: " IPSTR, IP2STR(&dns_info.ip.u_addr.ip4));
+            ESP_LOGI(TAG, "~~~~~~~~~~~~~~");
+
+            ESP_LOGI(TAG, "GOT ip event!!!");
+            xEventGroupSetBits(event_group, GOTIP_BIT);
+        } else if (event_id == IP_EVENT_PPP_LOST_IP) {
+            ESP_LOGI(TAG, "Modem Disconnect from PPP Server");
+        } else if (event_id == IP_EVENT_GOT_IP6) {
+            ESP_LOGI(TAG, "GOT IPv6 event!");
+            ip_event_got_ip6_t *event = (ip_event_got_ip6_t *)event_data;
+            ESP_LOGI(TAG, "Got IPv6 address " IPV6STR, IPV62STR(event->ip6_info.ip));
+        }
+    }
+}
+
+static void on_ppp_changed(void *arg, esp_event_base_t event_base,
+                           int32_t event_id, void *event_data)
+{
+    ESP_LOGI(TAG, "PPP state changed event %" PRId32, event_id);
+    if (event_id == NETIF_PPP_ERRORUSER) {
+        /* User interrupted event from esp-netif */
+        esp_netif_t *netif = (esp_netif_t *)event_data;
+        ESP_LOGI(TAG, "User interrupted event from netif:%p", netif);
+        xEventGroupSetBits(event_group, 2);
+    }
+}
+
+/* "pppd" command */
+static struct {
+    struct arg_str *control;
+    struct arg_end *end;
+} pppd_control_args;
+
+static int pppd_cmd_control(int argc, char **argv)
+{
+    int nerrors = arg_parse(argc, argv, (void **)&pppd_control_args);
+    if (nerrors != 0) {
+        arg_print_errors(stderr, pppd_control_args.end, argv[0]);
+        return 1;
+    }
+
+    if (!strncmp(pppd_control_args.control->sval[0], "info", 4)) {
+        esp_netif_ip_info_t ip;
+        printf("%s:\n", esp_netif_get_desc(s_ppp_netif));
+        esp_netif_get_ip_info(s_ppp_netif, &ip);
+        printf("  IP: " IPSTR "\r\n", IP2STR(&ip.ip));
+        printf("  MASK: " IPSTR "\r\n", IP2STR(&ip.netmask));
+        printf("  GW: " IPSTR "\r\n", IP2STR(&ip.gw));
+    }
+    return 0;
+}
+
+/* "iperf" command */
+
+static struct {
+    struct arg_str *ip;
+    struct arg_lit *server;
+    struct arg_lit *udp;
+    struct arg_lit *version;
+    struct arg_int *port;
+    struct arg_int *length;
+    struct arg_int *interval;
+    struct arg_int *time;
+    struct arg_int *bw_limit;
+    struct arg_lit *abort;
+    struct arg_end *end;
+} iperf_args;
+
+static int ppp_cmd_iperf(int argc, char **argv)
+{
+    int nerrors = arg_parse(argc, argv, (void **)&iperf_args);
+    iperf_cfg_t cfg;
+
+    if (nerrors != 0) {
+        arg_print_errors(stderr, iperf_args.end, argv[0]);
+        return 0;
+    }
+
+    memset(&cfg, 0, sizeof(cfg));
+
+    // ethernet iperf only support IPV4 address
+    cfg.type = IPERF_IP_TYPE_IPV4;
+
+    /* iperf -a */
+    if (iperf_args.abort->count != 0) {
+        iperf_stop();
+        return 0;
+    }
+
+    if (((iperf_args.ip->count == 0) && (iperf_args.server->count == 0)) ||
+            ((iperf_args.ip->count != 0) && (iperf_args.server->count != 0))) {
+        ESP_LOGE(__func__, "Wrong mode! ESP32 should run in client or server mode");
+        return 0;
+    }
+
+    /* iperf -s */
+    if (iperf_args.ip->count == 0) {
+        cfg.flag |= IPERF_FLAG_SERVER;
+    }
+    /* iperf -c SERVER_ADDRESS */
+    else {
+        cfg.destination_ip4 = esp_ip4addr_aton(iperf_args.ip->sval[0]);
+        cfg.flag |= IPERF_FLAG_CLIENT;
+    }
+
+    if (iperf_args.length->count == 0) {
+        cfg.len_send_buf = 0;
+    } else {
+        cfg.len_send_buf = iperf_args.length->ival[0];
+    }
+
+    /* wait for ip, could blocked here */
+    xEventGroupWaitBits(event_group, GOTIP_BIT, pdFALSE, pdTRUE, portMAX_DELAY);
+
+    cfg.source_ip4 = INADDR_ANY;
+
+    /* iperf -u */
+    if (iperf_args.udp->count == 0) {
+        cfg.flag |= IPERF_FLAG_TCP;
+    } else {
+        cfg.flag |= IPERF_FLAG_UDP;
+    }
+
+    /* iperf -p */
+    if (iperf_args.port->count == 0) {
+        cfg.sport = IPERF_DEFAULT_PORT;
+        cfg.dport = IPERF_DEFAULT_PORT;
+    } else {
+        if (cfg.flag & IPERF_FLAG_SERVER) {
+            cfg.sport = iperf_args.port->ival[0];
+            cfg.dport = IPERF_DEFAULT_PORT;
+        } else {
+            cfg.sport = IPERF_DEFAULT_PORT;
+            cfg.dport = iperf_args.port->ival[0];
+        }
+    }
+
+    /* iperf -i */
+    if (iperf_args.interval->count == 0) {
+        cfg.interval = IPERF_DEFAULT_INTERVAL;
+    } else {
+        cfg.interval = iperf_args.interval->ival[0];
+        if (cfg.interval <= 0) {
+            cfg.interval = IPERF_DEFAULT_INTERVAL;
+        }
+    }
+
+    /* iperf -t */
+    if (iperf_args.time->count == 0) {
+        cfg.time = IPERF_DEFAULT_TIME;
+    } else {
+        cfg.time = iperf_args.time->ival[0];
+        if (cfg.time <= cfg.interval) {
+            cfg.time = cfg.interval;
+        }
+    }
+
+    /* iperf -b */
+    if (iperf_args.bw_limit->count == 0) {
+        cfg.bw_lim = IPERF_DEFAULT_NO_BW_LIMIT;
+    } else {
+        cfg.bw_lim = iperf_args.bw_limit->ival[0];
+        if (cfg.bw_lim <= 0) {
+            cfg.bw_lim = IPERF_DEFAULT_NO_BW_LIMIT;
+        }
+    }
+
+    printf("mode=%s-%s sip=" IPSTR ":%" PRIu16 ", dip=%" PRIu32 ".%" PRIu32 ".%" PRIu32 ".%" PRIu32 ":%" PRIu16 ", interval=%" PRIu32 ", time=%" PRIu32 "\r\n",
+           cfg.flag & IPERF_FLAG_TCP ? "tcp" : "udp",
+           cfg.flag & IPERF_FLAG_SERVER ? "server" : "client",
+           (uint16_t) cfg.source_ip4 & 0xFF,
+           (uint16_t) (cfg.source_ip4 >> 8) & 0xFF,
+           (uint16_t) (cfg.source_ip4 >> 16) & 0xFF,
+           (uint16_t) (cfg.source_ip4 >> 24) & 0xFF,
+           cfg.sport,
+           cfg.destination_ip4 & 0xFF, (cfg.destination_ip4 >> 8) & 0xFF,
+           (cfg.destination_ip4 >> 16) & 0xFF, (cfg.destination_ip4 >> 24) & 0xFF, cfg.dport,
+           cfg.interval, cfg.time);
+
+    iperf_start(&cfg);
+    return 0;
+}
+
+void register_pppd(void)
+{
+    event_group = xEventGroupCreate();
+    ESP_ERROR_CHECK(esp_event_loop_create_default());
+    ESP_ERROR_CHECK(esp_netif_init());
+
+    esp_netif_config_t ppp_netif_config = ESP_NETIF_DEFAULT_PPP();
+    s_ppp_netif = esp_netif_new(&ppp_netif_config);
+    assert(s_ppp_netif);
+    esp_netif_ppp_config_t ppp_config = { true, true };
+    esp_netif_ppp_set_params(s_ppp_netif, &ppp_config);
+
+    ESP_ERROR_CHECK(modem_init_network(s_ppp_netif));
+    ESP_ERROR_CHECK(esp_event_handler_register(IP_EVENT, ESP_EVENT_ANY_ID, on_modem_event, NULL));
+    ESP_ERROR_CHECK(esp_event_handler_register(NETIF_PPP_STATUS, ESP_EVENT_ANY_ID, on_ppp_changed, NULL));
+
+    modem_start_network();
+
+    pppd_control_args.control = arg_str1(NULL, NULL, "<info>", "Get info of pppd");
+    pppd_control_args.end = arg_end(1);
+    const esp_console_cmd_t cmd = {
+        .command = "pppd",
+        .help = "PPP interface IO control",
+        .hint = NULL,
+        .func = pppd_cmd_control,
+        .argtable = &pppd_control_args
+    };
+    ESP_ERROR_CHECK(esp_console_cmd_register(&cmd));
+
+    iperf_args.ip = arg_str0("c", "client", "<ip>",
+                             "run in client mode, connecting to <host>");
+    iperf_args.server = arg_lit0("s", "server", "run in server mode");
+    iperf_args.udp = arg_lit0("u", "udp", "use UDP rather than TCP");
+    iperf_args.version = arg_lit0("V", "ipv6_domain", "use IPV6 address rather than IPV4");
+    iperf_args.port = arg_int0("p", "port", "<port>",
+                               "server port to listen on/connect to");
+    iperf_args.length = arg_int0("l", "len", "<length>", "set read/write buffer size");
+    iperf_args.interval = arg_int0("i", "interval", "<interval>",
+                                   "seconds between periodic bandwidth reports");
+    iperf_args.time = arg_int0("t", "time", "<time>", "time in seconds to transmit for (default 10 secs)");
+    iperf_args.bw_limit = arg_int0("b", "bandwidth", "<bandwidth>", "bandwidth to send at in Mbits/sec");
+    iperf_args.abort = arg_lit0("a", "abort", "abort running iperf");
+    iperf_args.end = arg_end(1);
+    const esp_console_cmd_t iperf_cmd = {
+        .command = "iperf",
+        .help = "iperf command",
+        .hint = NULL,
+        .func = &ppp_cmd_iperf,
+        .argtable = &iperf_args
+    };
+    ESP_ERROR_CHECK(esp_console_cmd_register(&iperf_cmd));
+}

components/esp_modem/test/target_iperf/main/network_dce.h

@@ -0,0 +1,26 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+
+#pragma once
+#include "esp_err.h"
+#include "esp_netif.h"
+
+/**
+ * @brief Initialize PPP network (with a generic PPP server)
+ * @param netif PPP netif
+ * @return ESP_OK on success
+ */
+esp_err_t modem_init_network(esp_netif_t *netif);
+
+/**
+ * @brief Start PPP network
+ */
+void modem_start_network();
+
+/**
+ * @brief Stop PPP network
+ */
+void modem_stop_network();

components/esp_modem/test/target_iperf/main/pppd_iperf_main.c

@@ -0,0 +1,43 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#include <stdio.h>
+#include <string.h>
+#include "sdkconfig.h"
+#include "esp_log.h"
+#include "esp_console.h"
+#include "cmd_system.h"
+
+void register_pppd(void);
+
+void app_main(void)
+{
+    esp_console_repl_t *repl = NULL;
+    esp_console_repl_config_t repl_config = ESP_CONSOLE_REPL_CONFIG_DEFAULT();
+    esp_console_dev_uart_config_t uart_config = ESP_CONSOLE_DEV_UART_CONFIG_DEFAULT();
+    repl_config.prompt = "iperf>";
+    // init console REPL environment
+    ESP_ERROR_CHECK(esp_console_new_repl_uart(&uart_config, &repl_config, &repl));
+
+    /* Register commands */
+    register_system_common();
+    register_pppd();
+
+    printf("\n =======================================================\n");
+    printf(" |       Steps to Test PPP Client Bandwidth            |\n");
+    printf(" |                                                     |\n");
+    printf(" |  1. Enter 'help', check all supported commands      |\n");
+    printf(" |  2. Start PPP server on host system                 |\n");
+    printf(" |     - pppd /dev/ttyUSB1 115200 192.168.11.1:192.168.11.2 modem local noauth debug nocrtscts nodetach +ipv6\n");
+    printf(" |  3. Wait ESP32 to get IP from PPP server            |\n");
+    printf(" |  4. Enter 'pppd info' (optional)                    |\n");
+    printf(" |  5. Server: 'iperf -u -s -i 3'                      |\n");
+    printf(" |  6. Client: 'iperf -u -c SERVER_IP -t 60 -i 3'      |\n");
+    printf(" |                                                     |\n");
+    printf(" =======================================================\n\n");
+
+    // start console REPL
+    ESP_ERROR_CHECK(esp_console_start_repl(repl));
+}

components/esp_modem/test/target_iperf/pytest_ppp_iperf.py

@@ -0,0 +1,112 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+"""
+Test case for iperf example.
+
+This test case might have problem running on Windows:
+
+- use `sudo killall iperf` to force kill iperf, didn't implement windows version
+
+"""
+from __future__ import division, unicode_literals
+
+import logging
+import os
+import subprocess
+
+import pytest
+from common_test_methods import get_host_ip4_by_dest_ip
+from idf_iperf_test_util import IperfUtility
+from pytest_embedded import Dut
+
+try:
+    from typing import Any, Callable, Tuple
+except ImportError:
+    # Only used for type annotations
+    pass
+
+NO_BANDWIDTH_LIMIT = -1  # iperf send bandwidth is not limited
+
+
+class IperfTestUtilityEth(IperfUtility.IperfTestUtility):
+    """ iperf test implementation """
+    def __init__(self, dut: str, config_name: str, pc_nic_ip: str, pc_iperf_log_file: str, test_result:Any=None) -> None:
+        IperfUtility.IperfTestUtility.__init__(self, dut, config_name, 'None', 'None', pc_nic_ip, pc_iperf_log_file, test_result)
+
+    def setup(self) -> Tuple[str,int]:
+        """
+        setup iperf test:
+
+        1. kill current iperf process
+        2. reboot DUT (currently iperf is not very robust, need to reboot DUT)
+        """
+        try:
+            subprocess.check_output('sudo killall iperf 2>&1 > /dev/null', shell=True)
+        except subprocess.CalledProcessError:
+            pass
+        self.dut.write('restart')
+        dut_ip = self.dut.expect(r'pppd_test: IP          : (\d+\.\d+\.\d+\.\d+)').group(1)
+        self.dut.expect("Type 'help' to get the list of commands.")
+        self.dut.expect('iperf>')
+        rssi = 0
+        return dut_ip, rssi
+
+
+@pytest.fixture
+def log_performance(record_property: Callable[[str, object], None]) -> Callable[[str, str], None]:
+    """
+    log performance item with pre-defined format to the console
+    and record it under the ``properties`` tag in the junit report if available.
+    """
+
+    def real_func(item: str, value: str) -> None:
+        """
+        :param item: performance item name
+        :param value: performance value
+        """
+        logging.info('[Performance][%s]: %s', item, value)
+
+    return real_func
+
+
+def test_esp_pppd_iperf(
+    dut: Dut,
+    log_performance: Callable[[str, object], None],
+    # check_performance: Callable[[str, float, str], None],
+) -> None:
+    """
+    steps: |
+      1. test TCP tx rx and UDP tx rx throughput
+      2. compare with the pre-defined pass standard
+    """
+
+    # 1. preparing
+    pc_iperf_log_file = os.path.join(dut.logdir, 'pc_iperf_log.md')
+    # 2. wait for DUT
+    dut_ip = dut.expect(r'pppd_test: IP          : (\d+\.\d+\.\d+\.\d+)').group(1)
+
+    pc_nic_ip = get_host_ip4_by_dest_ip(dut_ip)
+    test_result = {
+        'tcp_tx': IperfUtility.TestResult('tcp', 'tx', 'ppp_client'),
+        'tcp_rx': IperfUtility.TestResult('tcp', 'rx', 'ppp_client'),
+        'udp_tx': IperfUtility.TestResult('udp', 'tx', 'ppp_client'),
+        'udp_rx': IperfUtility.TestResult('udp', 'rx', 'ppp_client'),
+    }
+    test_utility = IperfTestUtilityEth(dut, 'ppp', pc_nic_ip, pc_iperf_log_file, test_result)
+
+    # 3. run test for TCP Tx, Rx and UDP Tx, Rx
+    test_utility.run_test('tcp', 'tx', 0, 5)
+    test_utility.run_test('tcp', 'rx', 0, 5)
+    test_utility.run_test('udp', 'tx', 0, 5)
+    test_utility.run_test('udp', 'rx', 0, 5)
+
+    # 4. print out performance details
+    for throughput_type in test_result:
+        print('{}_throughput'.format(throughput_type))
+        print(test_result[throughput_type])
+        print('---------------------------')
+
+    # 5. log performance summary
+    for throughput_type in test_result:
+        log_performance('{}_throughput'.format(throughput_type),
+                        '{:.02f} Mbps'.format(test_result[throughput_type].get_best_throughput()))

components/esp_modem/test/target_iperf/sdkconfig.defaults

@@ -0,0 +1,3 @@
+CONFIG_COMPILER_CXX_EXCEPTIONS=y
+CONFIG_LWIP_PPP_SUPPORT=y
+CONFIG_ESP_MAIN_TASK_STACK_SIZE=4096

components/esp_mqtt_cxx/.cz.yaml

@@ -3,6 +3,6 @@ commitizen:
   bump_message: 'bump(mqtt_cxx): $current_version -> $new_version'
   pre_bump_hooks: python ../../ci/changelog.py esp_mqtt_cxx
   tag_format: mqtt_cxx-v$version
-  version: 0.2.0
+  version: 0.3.0
   version_files:
   - idf_component.yml

components/esp_mqtt_cxx/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [0.3.0](https://github.com/espressif/esp-protocols/commits/mqtt_cxx-v0.3.0)
+
+### Bug Fixes
+
+- Incorrect documentation link ([aa4e9d57](https://github.com/espressif/esp-protocols/commit/aa4e9d57))
+- reference protocol_examples_common from IDF ([e6c0538d](https://github.com/espressif/esp-protocols/commit/e6c0538d))
+- specify override_path in example manifest files ([fa005c63](https://github.com/espressif/esp-protocols/commit/fa005c63))
+
 ## [0.2.0](https://github.com/espressif/esp-protocols/commits/mqtt_cxx-v0.2.0)
 
 ### Features

components/esp_mqtt_cxx/README.md

@@ -10,4 +10,4 @@ Get started with [examples](https://github.com/espressif/esp-protocols/tree/mast
 
 ## Documentation
 
-* View the full [html documentation](https://docs.espressif.com/projects/esp-protocols/docs/latest/esp_mqtt_cxx/index.html)
+* View the full [html documentation](https://docs.espressif.com/projects/esp-protocols/esp_mqtt_cxx/docs/latest/index.html)

components/esp_mqtt_cxx/idf_component.yml

@@ -1,4 +1,4 @@
-version: "0.2.0"
+version: "0.3.0"
 description: C++ APIs for ESP-MQTT library
 url: https://github.com/espressif/esp-protocols/tree/master/components/esp_mqtt_cxx
 issues: https://github.com/espressif/esp-protocols/issues

components/esp_websocket_client/.cz.yaml

@@ -3,6 +3,6 @@ commitizen:
   bump_message: 'bump(websocket): $current_version -> $new_version'
   pre_bump_hooks: python ../../ci/changelog.py esp_websocket_client
   tag_format: websocket-v$version
-  version: 1.2.1
+  version: 1.2.2
   version_files:
   - idf_component.yml

components/esp_websocket_client/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [1.2.2](https://github.com/espressif/esp-protocols/commits/websocket-v1.2.2)
+
+### Bug Fixes
+
+- continuation after FIN in websocket client (#460) ([774d1c75e6](https://github.com/espressif/esp-protocols/commit/774d1c75e6))
+- Re-applie refs to common comps idf_component.yml ([9fe44a4504](https://github.com/espressif/esp-protocols/commit/9fe44a4504))
+
 ## [1.2.1](https://github.com/espressif/esp-protocols/commits/websocket-v1.2.1)
 
 ### Bug Fixes

components/esp_websocket_client/idf_component.yml

@@ -1,4 +1,4 @@
-version: "1.2.1"
+version: "1.2.2"
 description: WebSocket protocol client for ESP-IDF
 url: https://github.com/espressif/esp-protocols/tree/master/components/esp_websocket_client
 dependencies:

components/mdns/.cz.yaml

@@ -3,6 +3,6 @@ commitizen:
   bump_message: 'bump(mdns): $current_version -> $new_version'
   pre_bump_hooks: python ../../ci/changelog.py mdns
   tag_format: mdns-v$version
-  version: 1.2.2
+  version: 1.2.3
   version_files:
   - idf_component.yml

components/mdns/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## [1.2.3](https://github.com/espressif/esp-protocols/commits/mdns-v1.2.3)
+
+### Bug Fixes
+
+- fixed CI issues for host and afl tests ([4be5efc84e](https://github.com/espressif/esp-protocols/commit/4be5efc84e))
+- fix copy delegated host addr ([19fb36000c](https://github.com/espressif/esp-protocols/commit/19fb36000c))
+- enable CONFIG_ESP_WIFI_ENABLED if CONFIG_SOC_WIFI_SUPPORTED is also enabled (for ESP-IDF <= 5.1) ([d20a718320](https://github.com/espressif/esp-protocols/commit/d20a718320))
+- remove protocol_examples_common ([1ee9dae6bf](https://github.com/espressif/esp-protocols/commit/1ee9dae6bf))
+- move the example into a subdirectory ([d28232b9f8](https://github.com/espressif/esp-protocols/commit/d28232b9f8))
+- reference protocol_examples_common from IDF ([c83b76ea8f](https://github.com/espressif/esp-protocols/commit/c83b76ea8f))
+
 ## [1.2.2](https://github.com/espressif/esp-protocols/commits/mdns-v1.2.2)
 
 ### Bug Fixes

components/mdns/idf_component.yml

@@ -1,4 +1,4 @@
-version: "1.2.2"
+version: "1.2.3"
 description: mDNS
 url: https://github.com/espressif/esp-protocols/tree/master/components/mdns
 dependencies:

components/mdns/mdns.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -132,7 +132,7 @@ static inline esp_netif_t *esp_netif_from_preset_if(mdns_predef_if_t predef_if)
         return esp_netif_get_handle_from_ifkey("WIFI_STA_DEF");
     case MDNS_IF_AP:
         return esp_netif_get_handle_from_ifkey("WIFI_AP_DEF");
-#if CONFIG_ETH_ENABLED
+#if CONFIG_ETH_ENABLED && CONFIG_MDNS_PREDEF_NETIF_ETH
     case MDNS_IF_ETH:
         return esp_netif_get_handle_from_ifkey("ETH_DEF");
 #endif
@@ -4189,7 +4189,7 @@ void mdns_preset_if_handle_system_event(void *arg, esp_event_base_t event_base,
         }
     } else
 #endif
-#if CONFIG_ETH_ENABLED
+#if CONFIG_ETH_ENABLED && CONFIG_MDNS_PREDEF_NETIF_ETH
         if (event_base == ETH_EVENT) {
             switch (event_id) {
             case ETHERNET_EVENT_CONNECTED:
@@ -4214,7 +4214,7 @@ void mdns_preset_if_handle_system_event(void *arg, esp_event_base_t event_base,
                     post_mdns_enable_pcb(MDNS_IF_STA, MDNS_IP_PROTOCOL_V4);
                     post_mdns_announce_pcb(MDNS_IF_STA, MDNS_IP_PROTOCOL_V6);
                     break;
-#if CONFIG_ETH_ENABLED
+#if CONFIG_ETH_ENABLED && CONFIG_MDNS_PREDEF_NETIF_ETH
                 case IP_EVENT_ETH_GOT_IP:
                     post_mdns_enable_pcb(MDNS_IF_ETH, MDNS_IP_PROTOCOL_V4);
                     break;

docs/esp_modem/en/README.rst

@@ -120,6 +120,7 @@ Switching between manual modes
 The diagram below depicts allowed transitions between manual CMUX modes
 
 ::
+
                              +------------------------------------
                              |                                   |
       +----------+    +-------------+    +------------+    +----------+
@@ -187,8 +188,18 @@ Is defined using standard configuration structures for ``DTE`` and
 Known issues
 ------------
 
-There are certain issues, especially in using CMUX mode which you can
-experience with some devices:
+There are certain typical issues you might experience when working with esp_modem. This section elaborates on these problems and offers workarounds.
+
+When using UART terminal, you might find running OTA over https quite unstable, exhibiting frequent UART buffer overflows.
+The reason for that is that ESP32's UART is interrupt driven, so while performing computational extensive tasks with TLS session,
+we need to timely interrupt to process incoming data. Here'are few suggestions to mitigate the impact:
+* Move UART ISR to IRAM
+* Increase internal UART rx buffer size
+* Increase UART terminal task priority
+* Use UART flow control
+If none of the above helps, you can check the test ``target_ota``, which performs OTA in two steps -- first read the data, then pass the data to mbedTLS. See the test ``README.md`` for more details.
+
+Potential issues when using CMUX mode and these devices:
 
 1) Some modems (e.g. A76xx serries) use 2 bytes CMUX payload, which
 might cause buffer overflow when trying to defragment the payload.

docs/esp_modem/en/advanced_api.rst

@@ -26,15 +26,17 @@ Create custom module
 
 Creating a custom module is necessary if the application needs to use a specific device that is not supported
 and their commands differ from any of the supported devices. In this case it is recommended to define a new class
-representing this specific device and derive from the :cpp:class:`esp_modem::GenericModule`. In order to instantiate
-the appropriate DCE of this module, application could use :ref:`the DCE factory<dce_factory>`, and build the DCE with
-the specific module, using :cpp:func:`esp_modem::dce_factory::Factory::build`.
+representing this specific device and derive from the :cpp:class:`esp_modem::GenericModule` (or any other available
+module, that's close to your custom device). Then you can create the DCE using :ref:`the DCE factory<dce_factory>`
+public method :cpp:func:`esp_modem::dce_factory::Factory::create_unique_dce_from`.
 
-Please refer to the implementation of the existing modules.
-
-Please note that the ``modem_console`` example defines a trivial custom modem DCE which overrides one command,
+Please note that the ``pppos_client`` example defines a trivial custom DCE which overrides one command, and adds a new command
 for demonstration purposes only.
 
+It is also possible to create a specific DCE class that would conform to the generic ``DCE`` API and use all generic commands,
+work with commands differently. This might be useful to add some custom preprocessing of commands or replies.
+Please check the ``modem_console`` example with ``CONFIG_EXAMPLE_MODEM_DEVICE_SHINY=y`` configuration which demonstrates
+overriding default ``command()`` method to implement URC processing in user space.
 
 Create new communication interface
 ----------------------------------

docs/requirements.txt

@@ -0,0 +1,8 @@
+sphinxcontrib-applehelp==1.0.4
+sphinxcontrib_devhelp==1.0.2
+sphinxcontrib-htmlhelp==2.0.1
+sphinxcontrib-serializinghtml==1.1.5
+sphinxcontrib-qthelp==1.0.3
+breathe==4.35
+recommonmark==0.7.1
+esp-docs==1.7.1

examples/esp_netif/slip_custom_netif/components/slip_modem/library/slip_modem_netif.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2019-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2019-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -109,7 +109,11 @@ static esp_netif_t *get_netif_with_esp_index(int index)
 {
     esp_netif_t *netif = NULL;
     int counter = 0;
+#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 2, 0)
+    while ((netif = esp_netif_next_unsafe(netif)) != NULL) {
+#else
     while ((netif = esp_netif_next(netif)) != NULL) {
+#endif
         if (counter == index) {
             return netif;
         }
@@ -124,7 +128,11 @@ static int get_esp_netif_index(esp_netif_t *esp_netif)
 {
     esp_netif_t *netif = NULL;
     int counter = 0;
+#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 2, 0)
+    while ((netif = esp_netif_next_unsafe(netif)) != NULL) {
+#else
     while ((netif = esp_netif_next(netif)) != NULL) {
+#endif
         if (esp_netif == netif) {
             return counter;
         }