Add certificate bundle capability to WiFiClientSecure (#6106)

* Add certificate bundle capability to WiFiClientSecure

Enable usage of the ESP32 IDF's certificate bundle for WiFiClientSecure connections.

Adds the ability to load a bundle or root certificates and use them for authenticating SSL servers.

Based on work from Onno-Dirkzwager, Duckle29, kubo6472, meltdown03, kinafu and others.

See also:
- https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/protocols/esp_crt_bundle.html
- https://github.com/espressif/arduino-esp32/issues/3646
- libraries/WiFiClientSecure/README.md

* Fix build issues

* Clean up old bundle index when NULL bundle is attached

libraries/WiFiClientSecure/src/WiFiClientSecure.cpp

@@ -19,6 +19,7 @@
 */
 
 #include "WiFiClientSecure.h"
+#include "esp_crt_bundle.h"
 #include <lwip/sockets.h>
 #include <lwip/netdb.h>
 #include <errno.h>
@@ -44,6 +45,7 @@ WiFiClientSecure::WiFiClientSecure()
     _psKey = NULL;
     next = NULL;
     _alpn_protos = NULL;
+    _use_ca_bundle = false;
 }
 
 
@@ -129,7 +131,7 @@ int WiFiClientSecure::connect(const char *host, uint16_t port, const char *CA_ce
     if(_timeout > 0){
         sslclient->handshake_timeout = _timeout;
     }
-    int ret = start_ssl_client(sslclient, host, port, _timeout, CA_cert, cert, private_key, NULL, NULL, _use_insecure, _alpn_protos);
+    int ret = start_ssl_client(sslclient, host, port, _timeout, CA_cert, _use_ca_bundle, cert, private_key, NULL, NULL, _use_insecure, _alpn_protos);
     _lastError = ret;
     if (ret < 0) {
         log_e("start_ssl_client: %d", ret);
@@ -149,7 +151,7 @@ int WiFiClientSecure::connect(const char *host, uint16_t port, const char *pskId
     if(_timeout > 0){
         sslclient->handshake_timeout = _timeout;
     }
-    int ret = start_ssl_client(sslclient, host, port, _timeout, NULL, NULL, NULL, pskIdent, psKey, _use_insecure, _alpn_protos);
+    int ret = start_ssl_client(sslclient, host, port, _timeout, NULL, false, NULL, NULL, pskIdent, psKey, _use_insecure, _alpn_protos);
     _lastError = ret;
     if (ret < 0) {
         log_e("start_ssl_client: %d", ret);
@@ -263,6 +265,18 @@ void WiFiClientSecure::setCACert (const char *rootCA)
     _CA_cert = rootCA;
 }
 
+ void WiFiClientSecure::setCACertBundle(const uint8_t * bundle)
+ {
+    if (bundle != NULL)
+    {
+        esp_crt_bundle_set(bundle);
+        _use_ca_bundle = true;
+    } else {
+        esp_crt_bundle_detach(NULL);
+        _use_ca_bundle = false;
+    }
+ }
+
 void WiFiClientSecure::setCertificate (const char *client_ca)
 {
     _cert = client_ca;

libraries/WiFiClientSecure/src/WiFiClientSecure.h

@@ -40,6 +40,7 @@ protected:
     const char *_pskIdent; // identity for PSK cipher suites
     const char *_psKey; // key in hex for PSK cipher suites
     const char **_alpn_protos;
+    bool _use_ca_bundle;
 
 public:
     WiFiClientSecure *next;
@@ -70,6 +71,7 @@ public:
     void setCertificate(const char *client_ca);
     void setPrivateKey (const char *private_key);
     bool loadCACert(Stream& stream, size_t size);
+    void setCACertBundle(const uint8_t * bundle);
     bool loadCertificate(Stream& stream, size_t size);
     bool loadPrivateKey(Stream& stream, size_t size);
     bool verify(const char* fingerprint, const char* domain_name);

libraries/WiFiClientSecure/src/esp_crt_bundle.c

@@ -0,0 +1,218 @@
+// Copyright 2018-2019 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+#include <string.h>
+#include <esp_system.h>
+#include <esp32-hal-log.h>
+#include "esp_crt_bundle.h"
+#include "esp_err.h"
+
+#define BUNDLE_HEADER_OFFSET 2
+#define CRT_HEADER_OFFSET 4
+
+static const char *TAG = "esp-x509-crt-bundle";
+
+/* a dummy certificate so that
+ * cacert_ptr passes non-NULL check during handshake */
+static mbedtls_x509_crt s_dummy_crt;
+
+
+typedef struct crt_bundle_t {
+    const uint8_t **crts;
+    uint16_t num_certs;
+    size_t x509_crt_bundle_len;
+} crt_bundle_t;
+
+static crt_bundle_t s_crt_bundle;
+
+static int esp_crt_verify_callback(void *buf, mbedtls_x509_crt *crt, int data, uint32_t *flags);
+static int esp_crt_check_signature(mbedtls_x509_crt *child, const uint8_t *pub_key_buf, size_t pub_key_len);
+
+
+static int esp_crt_check_signature(mbedtls_x509_crt *child, const uint8_t *pub_key_buf, size_t pub_key_len)
+{
+    int ret = 0;
+    mbedtls_x509_crt parent;
+    const mbedtls_md_info_t *md_info;
+    unsigned char hash[MBEDTLS_MD_MAX_SIZE];
+
+    mbedtls_x509_crt_init(&parent);
+
+    if ( (ret = mbedtls_pk_parse_public_key(&parent.pk, pub_key_buf, pub_key_len) ) != 0) {
+        log_e("PK parse failed with error %X", ret);
+        goto cleanup;
+    }
+
+
+    // Fast check to avoid expensive computations when not necessary
+    if (!mbedtls_pk_can_do(&parent.pk, child->sig_pk)) {
+        log_e("Simple compare failed");
+        ret = -1;
+        goto cleanup;
+    }
+
+    md_info = mbedtls_md_info_from_type(child->sig_md);
+    if ( (ret = mbedtls_md( md_info, child->tbs.p, child->tbs.len, hash )) != 0 ) {
+        log_e("Internal mbedTLS error %X", ret);
+        goto cleanup;
+    }
+
+    if ( (ret = mbedtls_pk_verify_ext( child->sig_pk, child->sig_opts, &parent.pk,
+                                       child->sig_md, hash, mbedtls_md_get_size( md_info ),
+                                       child->sig.p, child->sig.len )) != 0 ) {
+
+        log_e("PK verify failed with error %X", ret);
+        goto cleanup;
+    }
+cleanup:
+    mbedtls_x509_crt_free(&parent);
+
+    return ret;
+}
+
+
+/* This callback is called for every certificate in the chain. If the chain
+ * is proper each intermediate certificate is validated through its parent
+ * in the x509_crt_verify_chain() function. So this callback should
+ * only verify the first untrusted link in the chain is signed by the
+ * root certificate in the trusted bundle
+*/
+int esp_crt_verify_callback(void *buf, mbedtls_x509_crt *crt, int depth, uint32_t *flags)
+{
+    mbedtls_x509_crt *child = crt;
+
+    /* It's OK for a trusted cert to have a weak signature hash alg.
+       as we already trust this certificate */
+    uint32_t flags_filtered = *flags & ~(MBEDTLS_X509_BADCERT_BAD_MD);
+
+    if (flags_filtered != MBEDTLS_X509_BADCERT_NOT_TRUSTED) {
+        return 0;
+    }
+
+
+    if (s_crt_bundle.crts == NULL) {
+        log_e("No certificates in bundle");
+        return MBEDTLS_ERR_X509_FATAL_ERROR;
+    }
+
+    log_d("%d certificates in bundle", s_crt_bundle.num_certs);
+
+    size_t name_len = 0;
+    const uint8_t *crt_name;
+
+    bool crt_found = false;
+    int start = 0;
+    int end = s_crt_bundle.num_certs - 1;
+    int middle = (end - start) / 2;
+
+    /* Look for the certificate using binary search on subject name */
+    while (start <= end) {
+        name_len = s_crt_bundle.crts[middle][0] << 8 | s_crt_bundle.crts[middle][1];
+        crt_name = s_crt_bundle.crts[middle] + CRT_HEADER_OFFSET;
+
+        int cmp_res = memcmp(child->issuer_raw.p, crt_name, name_len );
+        if (cmp_res == 0) {
+            crt_found = true;
+            break;
+        } else if (cmp_res < 0) {
+            end = middle - 1;
+        } else {
+            start = middle + 1;
+        }
+        middle = (start + end) / 2;
+    }
+
+    int ret = MBEDTLS_ERR_X509_FATAL_ERROR;
+    if (crt_found) {
+        size_t key_len = s_crt_bundle.crts[middle][2] << 8 | s_crt_bundle.crts[middle][3];
+        ret = esp_crt_check_signature(child, s_crt_bundle.crts[middle] + CRT_HEADER_OFFSET + name_len, key_len);
+    }
+
+    if (ret == 0) {
+        log_i("Certificate validated");
+        *flags = 0;
+        return 0;
+    }
+
+    log_e("Failed to verify certificate");
+    return MBEDTLS_ERR_X509_FATAL_ERROR;
+}
+
+
+/* Initialize the bundle into an array so we can do binary search for certs,
+   the bundle generated by the python utility is already presorted by subject name
+ */
+static esp_err_t esp_crt_bundle_init(const uint8_t *x509_bundle)
+{
+    s_crt_bundle.num_certs = (x509_bundle[0] << 8) | x509_bundle[1];
+    s_crt_bundle.crts = calloc(s_crt_bundle.num_certs, sizeof(x509_bundle));
+
+    if (s_crt_bundle.crts == NULL) {
+        log_e("Unable to allocate memory for bundle");
+        return ESP_ERR_NO_MEM;
+    }
+
+    const uint8_t *cur_crt;
+    cur_crt = x509_bundle + BUNDLE_HEADER_OFFSET;
+
+    for (int i = 0; i < s_crt_bundle.num_certs; i++) {
+        s_crt_bundle.crts[i] = cur_crt;
+
+        size_t name_len = cur_crt[0] << 8 | cur_crt[1];
+        size_t key_len = cur_crt[2] << 8 | cur_crt[3];
+        cur_crt = cur_crt + CRT_HEADER_OFFSET + name_len + key_len;
+    }
+
+    return ESP_OK;
+}
+
+esp_err_t esp_crt_bundle_attach(void *conf)
+{
+    esp_err_t ret = ESP_OK;
+    // If no bundle has been set by the user then use the bundle embedded in the binary
+    if (s_crt_bundle.crts == NULL) {
+        log_e("Failed to attach bundle");
+        return ret;
+    }
+
+    if (conf) {
+        /* point to a dummy certificate
+         * This is only required so that the
+         * cacert_ptr passes non-NULL check during handshake
+         */
+        mbedtls_ssl_config *ssl_conf = (mbedtls_ssl_config *)conf;
+        mbedtls_x509_crt_init(&s_dummy_crt);
+        mbedtls_ssl_conf_ca_chain(ssl_conf, &s_dummy_crt, NULL);
+        mbedtls_ssl_conf_verify(ssl_conf, esp_crt_verify_callback, NULL);
+    }
+
+    return ret;
+}
+
+void esp_crt_bundle_detach(mbedtls_ssl_config *conf)
+{
+    free(s_crt_bundle.crts);
+    s_crt_bundle.crts = NULL;
+    if (conf) {
+        mbedtls_ssl_conf_verify(conf, NULL, NULL);
+    }
+}
+
+void esp_crt_bundle_set(const uint8_t *x509_bundle)
+{
+    // Free any previously used bundle
+    free(s_crt_bundle.crts);
+    esp_crt_bundle_init(x509_bundle);
+}

libraries/WiFiClientSecure/src/esp_crt_bundle.h

@@ -0,0 +1,68 @@
+// Copyright 2017-2019 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+#ifndef _ESP_CRT_BUNDLE_H_
+#define _ESP_CRT_BUNDLE_H_
+
+#include "mbedtls/ssl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/**
+ * @brief      Attach and enable use of a bundle for certificate verification
+ *
+ * Attach and enable use of a bundle for certificate verification through a verification callback.
+ * If no specific bundle has been set through esp_crt_bundle_set() it will default to the
+ * bundle defined in menuconfig and embedded in the binary.
+ *
+ * @param[in]  conf      The config struct for the SSL connection.
+ *
+ * @return
+ *             - ESP_OK  if adding certificates was successful.
+ *             - Other   if an error occured or an action must be taken by the calling process.
+ */
+esp_err_t esp_crt_bundle_attach(void *conf);
+
+
+/**
+ * @brief      Disable and dealloc the certification bundle
+ *
+ * Removes the certificate verification callback and deallocates used resources
+ *
+ * @param[in]  conf      The config struct for the SSL connection.
+ */
+void esp_crt_bundle_detach(mbedtls_ssl_config *conf);
+
+
+/**
+ * @brief      Set the default certificate bundle used for verification
+ *
+ * Overrides the default certificate bundle. In most use cases the bundle should be
+ * set through menuconfig. The bundle needs to be sorted by subject name since binary search is
+ * used to find certificates.
+ *
+ * @param[in]  x509_bundle     A pointer to the certificate bundle.
+ */
+void esp_crt_bundle_set(const uint8_t *x509_bundle);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif //_ESP_CRT_BUNDLE_H_

libraries/WiFiClientSecure/src/ssl_client.cpp

@@ -17,6 +17,7 @@
 #include <algorithm>
 #include <string>
 #include "ssl_client.h"
+#include "esp_crt_bundle.h"
 #include "WiFi.h"
 
 #ifndef MBEDTLS_KEY_EXCHANGE__SOME__PSK_ENABLED
@@ -53,14 +54,14 @@ void ssl_init(sslclient_context *ssl_client)
 }
 
 
-int start_ssl_client(sslclient_context *ssl_client, const char *host, uint32_t port, int timeout, const char *rootCABuff, const char *cli_cert, const char *cli_key, const char *pskIdent, const char *psKey, bool insecure, const char **alpn_protos)
+int start_ssl_client(sslclient_context *ssl_client, const char *host, uint32_t port, int timeout, const char *rootCABuff, bool useRootCABundle, const char *cli_cert, const char *cli_key, const char *pskIdent, const char *psKey, bool insecure, const char **alpn_protos)
 {
     char buf[512];
     int ret, flags;
     int enable = 1;
     log_v("Free internal heap before TLS %u", ESP.getFreeHeap());
 
-    if (rootCABuff == NULL && pskIdent == NULL && psKey == NULL && !insecure) {
+    if (rootCABuff == NULL && pskIdent == NULL && psKey == NULL && !insecure && !useRootCABundle) {
         return -1;
     }
 
@@ -183,6 +184,13 @@ int start_ssl_client(sslclient_context *ssl_client, const char *host, uint32_t p
             mbedtls_x509_crt_free(&ssl_client->ca_cert);
             return handle_error(ret);
         }
+    } else if (useRootCABundle) {
+        log_v("Attaching root CA cert bundle");
+        ret = esp_crt_bundle_attach(&ssl_client->ssl_conf);
+
+        if (ret < 0) {
+            return handle_error(ret);
+        }
     } else if (pskIdent != NULL && psKey != NULL) {
         log_v("Setting up PSK");
         // convert PSK from hex to binary

libraries/WiFiClientSecure/src/ssl_client.h

@@ -29,7 +29,7 @@ typedef struct sslclient_context {
 
 
 void ssl_init(sslclient_context *ssl_client);
-int start_ssl_client(sslclient_context *ssl_client, const char *host, uint32_t port, int timeout, const char *rootCABuff, const char *cli_cert, const char *cli_key, const char *pskIdent, const char *psKey, bool insecure, const char **alpn_protos);
+int start_ssl_client(sslclient_context *ssl_client, const char *host, uint32_t port, int timeout, const char *rootCABuff, bool useRootCABundle, const char *cli_cert, const char *cli_key, const char *pskIdent, const char *psKey, bool insecure, const char **alpn_protos);
 void stop_ssl_socket(sslclient_context *ssl_client, const char *rootCABuff, const char *cli_cert, const char *cli_key);
 int data_to_read(sslclient_context *ssl_client);
 int send_ssl_data(sslclient_context *ssl_client, const uint8_t *data, size_t len);