From 167618d6a4e2cfcaab967ba2ba920df64fb4795d Mon Sep 17 00:00:00 2001 From: Laukik Hase Date: Wed, 25 Jan 2023 16:45:01 +0530 Subject: [PATCH] examples/protocols: Added URI encoding/decoding feature - http_server/simple: Decoding received query - esp_http_client: Sending encoded query --- .../protocol_examples_common/CMakeLists.txt | 3 +- .../include/protocol_examples_utils.h | 49 +++ .../protocol_examples_utils.c | 388 ++++++++++++++++++ .../main/esp_http_client_example.c | 31 ++ .../protocols/http_server/simple/main/main.c | 11 +- .../simple/pytest_http_server_simple.py | 19 +- tools/ci/check_copyright_config.yaml | 10 + 7 files changed, 500 insertions(+), 11 deletions(-) create mode 100644 examples/common_components/protocol_examples_common/include/protocol_examples_utils.h create mode 100644 examples/common_components/protocol_examples_common/protocol_examples_utils.c diff --git a/examples/common_components/protocol_examples_common/CMakeLists.txt b/examples/common_components/protocol_examples_common/CMakeLists.txt index f155329bfd..15981d027c 100644 --- a/examples/common_components/protocol_examples_common/CMakeLists.txt +++ b/examples/common_components/protocol_examples_common/CMakeLists.txt @@ -1,7 +1,8 @@ set(srcs "stdin_out.c" "addr_from_stdin.c" "connect.c" - "wifi_connect.c") + "wifi_connect.c" + "protocol_examples_utils.c") if(CONFIG_EXAMPLE_PROVIDE_WIFI_CONSOLE_CMD) list(APPEND srcs "console_cmd.c") diff --git a/examples/common_components/protocol_examples_common/include/protocol_examples_utils.h b/examples/common_components/protocol_examples_common/include/protocol_examples_utils.h new file mode 100644 index 0000000000..79031171dc --- /dev/null +++ b/examples/common_components/protocol_examples_common/include/protocol_examples_utils.h @@ -0,0 +1,49 @@ +/* + * Utility functions for protocol examples + * + * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Unlicense OR CC0-1.0 + */ + +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +/** + * @brief Encode an URI + * + * @param dest a destination memory location + * @param src the source string + * @param len the length of the source string + * @return uint32_t the count of escaped characters + * + * @note Please allocate the destination buffer keeping in mind that encoding a + * special character will take up 3 bytes (for '%' and two hex digits). + * In the worst-case scenario, the destination buffer will have to be 3 times + * that of the source string. + */ +uint32_t example_uri_encode(char *dest, const char *src, size_t len); + +/** + * @brief Decode an URI + * + * @param dest a destination memory location + * @param src the source string + * @param len the length of the source string + * + * @note Please allocate the destination buffer keeping in mind that a decoded + * special character will take up 2 less bytes than its encoded form. + * In the worst-case scenario, the destination buffer will have to be + * the same size that of the source string. + */ +void example_uri_decode(char *dest, const char *src, size_t len); + +#ifdef __cplusplus +} +#endif diff --git a/examples/common_components/protocol_examples_common/protocol_examples_utils.c b/examples/common_components/protocol_examples_common/protocol_examples_utils.c new file mode 100644 index 0000000000..e6e3de5f62 --- /dev/null +++ b/examples/common_components/protocol_examples_common/protocol_examples_utils.c @@ -0,0 +1,388 @@ +/* + * Utility functions for protocol examples + * + * SPDX-FileCopyrightText: 2002-2021 Igor Sysoev + * 2011-2022 Nginx, Inc. + * + * SPDX-License-Identifier: BSD-2-Clause + * + * SPDX-FileContributor: 2023 Espressif Systems (Shanghai) CO LTD + */ +/* + * Copyright (C) 2002-2021 Igor Sysoev + * Copyright (C) 2011-2022 Nginx, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "protocol_examples_utils.h" + +/* Type of Escape algorithms to be used */ +#define NGX_ESCAPE_URI (0) +#define NGX_ESCAPE_ARGS (1) +#define NGX_ESCAPE_URI_COMPONENT (2) +#define NGX_ESCAPE_HTML (3) +#define NGX_ESCAPE_REFRESH (4) +#define NGX_ESCAPE_MEMCACHED (5) +#define NGX_ESCAPE_MAIL_AUTH (6) + +/* Type of Unescape algorithms to be used */ +#define NGX_UNESCAPE_URI (1) +#define NGX_UNESCAPE_REDIRECT (2) + + +uintptr_t ngx_escape_uri(u_char *dst, u_char *src, size_t size, unsigned int type) +{ + unsigned int n; + uint32_t *escape; + static u_char hex[] = "0123456789ABCDEF"; + + /* + * Per RFC 3986 only the following chars are allowed in URIs unescaped: + * + * unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + * gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" + * sub-delims = "!" / "$" / "&" / "'" / "(" / ")" + * / "*" / "+" / "," / ";" / "=" + * + * And "%" can appear as a part of escaping itself. The following + * characters are not allowed and need to be escaped: %00-%1F, %7F-%FF, + * " ", """, "<", ">", "\", "^", "`", "{", "|", "}". + */ + + /* " ", "#", "%", "?", not allowed */ + + static uint32_t uri[] = { + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + + /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ + 0xd000002d, /* 1101 0000 0000 0000 0000 0000 0010 1101 */ + + /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ + 0x50000000, /* 0101 0000 0000 0000 0000 0000 0000 0000 */ + + /* ~}| {zyx wvut srqp onml kjih gfed cba` */ + 0xb8000001, /* 1011 1000 0000 0000 0000 0000 0000 0001 */ + + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + }; + + /* " ", "#", "%", "&", "+", ";", "?", not allowed */ + + static uint32_t args[] = { + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + + /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ + 0xd800086d, /* 1101 1000 0000 0000 0000 1000 0110 1101 */ + + /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ + 0x50000000, /* 0101 0000 0000 0000 0000 0000 0000 0000 */ + + /* ~}| {zyx wvut srqp onml kjih gfed cba` */ + 0xb8000001, /* 1011 1000 0000 0000 0000 0000 0000 0001 */ + + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + }; + + /* not ALPHA, DIGIT, "-", ".", "_", "~" */ + + static uint32_t uri_component[] = { + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + + /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ + 0xfc009fff, /* 1111 1100 0000 0000 1001 1111 1111 1111 */ + + /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ + 0x78000001, /* 0111 1000 0000 0000 0000 0000 0000 0001 */ + + /* ~}| {zyx wvut srqp onml kjih gfed cba` */ + 0xb8000001, /* 1011 1000 0000 0000 0000 0000 0000 0001 */ + + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + }; + + /* " ", "#", """, "%", "'", not allowed */ + + static uint32_t html[] = { + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + + /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ + 0x500000ad, /* 0101 0000 0000 0000 0000 0000 1010 1101 */ + + /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ + 0x50000000, /* 0101 0000 0000 0000 0000 0000 0000 0000 */ + + /* ~}| {zyx wvut srqp onml kjih gfed cba` */ + 0xb8000001, /* 1011 1000 0000 0000 0000 0000 0000 0001 */ + + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + }; + + /* " ", """, "'", not allowed */ + + static uint32_t refresh[] = { + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + + /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ + 0x50000085, /* 0101 0000 0000 0000 0000 0000 1000 0101 */ + + /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ + 0x50000000, /* 0101 0000 0000 0000 0000 0000 0000 0000 */ + + /* ~}| {zyx wvut srqp onml kjih gfed cba` */ + 0xd8000001, /* 1011 1000 0000 0000 0000 0000 0000 0001 */ + + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + }; + + /* " ", "%", %00-%1F */ + + static uint32_t memcached[] = { + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + + /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ + 0x00000021, /* 0000 0000 0000 0000 0000 0000 0010 0001 */ + + /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + + /* ~}| {zyx wvut srqp onml kjih gfed cba` */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + }; + + /* mail_auth is the same as memcached */ + + static uint32_t *map[] = + { uri, args, uri_component, html, refresh, memcached, memcached }; + + + escape = map[type]; + + if (dst == NULL) { + + /* find the number of the characters to be escaped */ + + n = 0; + + while (size) { + if (escape[*src >> 5] & (1U << (*src & 0x1f))) { + n++; + } + src++; + size--; + } + + return (uintptr_t) n; + } + + while (size) { + if (escape[*src >> 5] & (1U << (*src & 0x1f))) { + *dst++ = '%'; + *dst++ = hex[*src >> 4]; + *dst++ = hex[*src & 0xf]; + src++; + + } else { + *dst++ = *src++; + } + size--; + } + + return (uintptr_t) dst; +} + + +void ngx_unescape_uri(u_char **dst, u_char **src, size_t size, unsigned int type) +{ + u_char *d, *s, ch, c, decoded; + enum { + sw_usual = 0, + sw_quoted, + sw_quoted_second + } state; + + d = *dst; + s = *src; + + state = 0; + decoded = 0; + + while (size--) { + + ch = *s++; + + switch (state) { + case sw_usual: + if (ch == '?' + && (type & (NGX_UNESCAPE_URI | NGX_UNESCAPE_REDIRECT))) { + *d++ = ch; + goto done; + } + + if (ch == '%') { + state = sw_quoted; + break; + } + + *d++ = ch; + break; + + case sw_quoted: + + if (ch >= '0' && ch <= '9') { + decoded = (u_char) (ch - '0'); + state = sw_quoted_second; + break; + } + + c = (u_char) (ch | 0x20); + if (c >= 'a' && c <= 'f') { + decoded = (u_char) (c - 'a' + 10); + state = sw_quoted_second; + break; + } + + /* the invalid quoted character */ + + state = sw_usual; + + *d++ = ch; + + break; + + case sw_quoted_second: + + state = sw_usual; + + if (ch >= '0' && ch <= '9') { + ch = (u_char) ((decoded << 4) + (ch - '0')); + + if (type & NGX_UNESCAPE_REDIRECT) { + if (ch > '%' && ch < 0x7f) { + *d++ = ch; + break; + } + + *d++ = '%'; *d++ = *(s - 2); *d++ = *(s - 1); + + break; + } + + *d++ = ch; + + break; + } + + c = (u_char) (ch | 0x20); + if (c >= 'a' && c <= 'f') { + ch = (u_char) ((decoded << 4) + (c - 'a') + 10); + + if (type & NGX_UNESCAPE_URI) { + if (ch == '?') { + *d++ = ch; + goto done; + } + + *d++ = ch; + break; + } + + if (type & NGX_UNESCAPE_REDIRECT) { + if (ch == '?') { + *d++ = ch; + goto done; + } + + if (ch > '%' && ch < 0x7f) { + *d++ = ch; + break; + } + + *d++ = '%'; *d++ = *(s - 2); *d++ = *(s - 1); + break; + } + + *d++ = ch; + + break; + } + + /* the invalid quoted character */ + + break; + } + } + +done: + + *dst = d; + *src = s; +} + + +uint32_t example_uri_encode(char *dest, const char *src, size_t len) +{ + if (!src || !dest) { + return 0; + } + + uintptr_t ret = ngx_escape_uri((unsigned char *)dest, (unsigned char *)src, len, NGX_ESCAPE_URI_COMPONENT); + return (uint32_t)(ret - (uintptr_t)dest); +} + + +void example_uri_decode(char *dest, const char *src, size_t len) +{ + if (!src || !dest) { + return; + } + + unsigned char *src_ptr = (unsigned char *)src; + unsigned char *dst_ptr = (unsigned char *)dest; + ngx_unescape_uri(&dst_ptr, &src_ptr, len, NGX_UNESCAPE_URI); +} diff --git a/examples/protocols/esp_http_client/main/esp_http_client_example.c b/examples/protocols/esp_http_client/main/esp_http_client_example.c index 99e7719072..50cebca86b 100644 --- a/examples/protocols/esp_http_client/main/esp_http_client_example.c +++ b/examples/protocols/esp_http_client/main/esp_http_client_example.c @@ -10,11 +10,13 @@ #include #include #include +#include #include "esp_log.h" #include "nvs_flash.h" #include "esp_event.h" #include "esp_netif.h" #include "protocol_examples_common.h" +#include "protocol_examples_utils.h" #include "esp_tls.h" #if CONFIG_MBEDTLS_CERTIFICATE_BUNDLE #include "esp_crt_bundle.h" @@ -431,6 +433,34 @@ static void https_with_hostname_path(void) esp_http_client_cleanup(client); } +static void http_encoded_query(void) +{ + esp_http_client_config_t config = { + .host = "httpbin.org", + .path = "/get", + .event_handler = _http_event_handler, + }; + + static const char query_val[] = "ABC xyz!012@#%&"; + char query_val_enc[64] = {0}; + + uint32_t enc_len = example_uri_encode(query_val_enc, query_val, strlen(query_val)); + if (enc_len > 0) { + ESP_LOG_BUFFER_HEXDUMP(TAG, query_val_enc, enc_len, ESP_LOG_DEBUG); + config.query = query_val_enc; + } + + esp_http_client_handle_t client = esp_http_client_init(&config); + esp_err_t err = esp_http_client_perform(client); + if (err == ESP_OK) { + ESP_LOGI(TAG, "HTTP GET Status = %d, content_length = %"PRIu64, + esp_http_client_get_status_code(client), + esp_http_client_get_content_length(client)); + } else { + ESP_LOGE(TAG, "HTTP GET request failed: %s", esp_err_to_name(err)); + } +} + static void http_relative_redirect(void) { esp_http_client_config_t config = { @@ -743,6 +773,7 @@ static void http_test_task(void *pvParameters) #if CONFIG_ESP_HTTP_CLIENT_ENABLE_DIGEST_AUTH http_auth_digest(); #endif + http_encoded_query(); http_relative_redirect(); http_absolute_redirect(); http_absolute_redirect_manual(); diff --git a/examples/protocols/http_server/simple/main/main.c b/examples/protocols/http_server/simple/main/main.c index 1ec5757c08..eddc0cb9e4 100644 --- a/examples/protocols/http_server/simple/main/main.c +++ b/examples/protocols/http_server/simple/main/main.c @@ -17,9 +17,12 @@ #include "esp_netif.h" #include "esp_eth.h" #include "protocol_examples_common.h" +#include "protocol_examples_utils.h" #include "esp_tls_crypto.h" #include +#define EXAMPLE_HTTP_QUERY_KEY_MAX_LEN (64) + /* A simple example that demonstrates how to create GET and POST * handlers for the web server. */ @@ -188,16 +191,22 @@ static esp_err_t hello_get_handler(httpd_req_t *req) buf = malloc(buf_len); if (httpd_req_get_url_query_str(req, buf, buf_len) == ESP_OK) { ESP_LOGI(TAG, "Found URL query => %s", buf); - char param[32]; + char param[EXAMPLE_HTTP_QUERY_KEY_MAX_LEN], dec_param[EXAMPLE_HTTP_QUERY_KEY_MAX_LEN] = {0}; /* Get value of expected key from query string */ if (httpd_query_key_value(buf, "query1", param, sizeof(param)) == ESP_OK) { ESP_LOGI(TAG, "Found URL query parameter => query1=%s", param); + example_uri_decode(dec_param, param, strnlen(param, EXAMPLE_HTTP_QUERY_KEY_MAX_LEN)); + ESP_LOGI(TAG, "Decoded query parameter => %s", dec_param); } if (httpd_query_key_value(buf, "query3", param, sizeof(param)) == ESP_OK) { ESP_LOGI(TAG, "Found URL query parameter => query3=%s", param); + example_uri_decode(dec_param, param, strnlen(param, EXAMPLE_HTTP_QUERY_KEY_MAX_LEN)); + ESP_LOGI(TAG, "Decoded query parameter => %s", dec_param); } if (httpd_query_key_value(buf, "query2", param, sizeof(param)) == ESP_OK) { ESP_LOGI(TAG, "Found URL query parameter => query2=%s", param); + example_uri_decode(dec_param, param, strnlen(param, EXAMPLE_HTTP_QUERY_KEY_MAX_LEN)); + ESP_LOGI(TAG, "Decoded query parameter => %s", dec_param); } } free(buf); diff --git a/examples/protocols/http_server/simple/pytest_http_server_simple.py b/examples/protocols/http_server/simple/pytest_http_server_simple.py index 4bab53fa76..0827bdb60e 100644 --- a/examples/protocols/http_server/simple/pytest_http_server_simple.py +++ b/examples/protocols/http_server/simple/pytest_http_server_simple.py @@ -115,17 +115,18 @@ def test_examples_protocol_http_server_simple(dut: Dut) -> None: if not client.test_post_handler(got_ip, got_port, random_data): raise RuntimeError - query = 'http://foobar' - logging.info('Test /hello with custom query : {}'.format(query)) - if not client.test_custom_uri_query(got_ip, got_port, query): + queries = 'query1=http%3A%2F%2Ffoobar&query3=abcd%2B1234%20xyz&query2=Esp%21%40%20%23%2471' + logging.info('Test /hello with custom query') + if not client.test_custom_uri_query(got_ip, got_port, queries): raise RuntimeError - dut.expect('Found URL query => ' + query, timeout=30) - query = 'abcd+1234%20xyz' - logging.info('Test /hello with custom query : {}'.format(query)) - if not client.test_custom_uri_query(got_ip, got_port, query): - raise RuntimeError - dut.expect_exact('Found URL query => ' + query, timeout=30) + dut.expect_exact('Found URL query => query1=http%3A%2F%2Ffoobar&query3=abcd%2B1234%20xyz&query2=Esp%21%40%20%23%2471', timeout=30) + dut.expect_exact('Found URL query parameter => query1=http%3A%2F%2Ffoobar', timeout=30) + dut.expect_exact('Decoded query parameter => http://foobar', timeout=30) + dut.expect_exact('Found URL query parameter => query3=abcd%2B1234%20xyz', timeout=30) + dut.expect_exact('Decoded query parameter => abcd+1234 xyz', timeout=30) + dut.expect_exact('Found URL query parameter => query2=Esp%21%40%20%23%2471', timeout=30) + dut.expect_exact('Decoded query parameter => Esp!@ #$71', timeout=30) @pytest.mark.esp32 diff --git a/tools/ci/check_copyright_config.yaml b/tools/ci/check_copyright_config.yaml index ff6709e25b..1a3957c825 100644 --- a/tools/ci/check_copyright_config.yaml +++ b/tools/ci/check_copyright_config.yaml @@ -156,6 +156,16 @@ tinyusb: - Unlicense - CC0-1.0 +protocol_examples_common_component: + include: + - 'examples/common_components/protocol_examples_common/' + allowed_licenses: + - Apache-2.0 + - BSD-2-Clause # Contains URI encoding/decoding code from nginx + - Unlicense + - CC0-1.0 + license_for_new_files: Unlicense OR CC0-1.0 + # files matching this section do not perform the check # file patterns starting with ! are negated, meaning files matching them won't match the section. ignore: