Merge branch 'bugfix/mqtt_fragmented_packed_msg_3.2' into 'release/v3.2'

MQTT support for fragmented and packed messages (Backport 3.2)

See merge request idf/esp-idf!4593

.gitlab-ci.yml

@@ -729,8 +729,11 @@ check_ut_cmake_make:
 
 check_submodule_sync:
   <<: *check_job_template
+  tags:
+    - github_sync
   variables:
     GIT_STRATEGY: clone
+  retry: 2
   script:
     # check if all submodules are correctly synced to public repostory
     - git submodule update --init --recursive

components/bootloader_support/src/bootloader_clock.c

@@ -31,14 +31,14 @@ void bootloader_clock_configure()
     /* Set CPU to 80MHz. Keep other clocks unmodified. */
     int cpu_freq_mhz = 80;
 
-    /* On ESP32 rev 0, switching to 80MHz if clock was previously set to
+    /* On ESP32 rev 0, switching to 80/160 MHz if clock was previously set to
      * 240 MHz may cause the chip to lock up (see section 3.5 of the errata
-     * document). For rev. 0, switch to 240 instead if it was chosen in
-     * menuconfig.
+     * document). For rev. 0, switch to 240 instead if it has been enabled
+     * previously.
      */
     uint32_t chip_ver_reg = REG_READ(EFUSE_BLK0_RDATA3_REG);
     if ((chip_ver_reg & EFUSE_RD_CHIP_VER_REV1_M) == 0 &&
-            CONFIG_ESP32_DEFAULT_CPU_FREQ_MHZ == 240) {
+            DPORT_REG_GET_FIELD(DPORT_CPU_PER_CONF_REG, DPORT_CPUPERIOD_SEL) == DPORT_CPUPERIOD_SEL_240) {
         cpu_freq_mhz = 240;
     }
 

components/bt/Kconfig

@@ -243,6 +243,39 @@ config MESH_DUPLICATE_SCAN_CACHE_SIZE
         Maximum number of adv packets which can be recorded in duplicate scan cache for BLE Mesh.
         When the maximum amount of device in the filter is reached, the cache will be refreshed.
 
+config BLE_ADV_REPORT_FLOW_CONTROL_SUPPORTED
+    bool "BLE adv report flow control supported"
+    depends on (BTDM_CONTROLLER_MODE_BTDM || BTDM_CONTROLLER_MODE_BLE_ONLY)
+    default y
+    help
+        The function is mainly used to enable flow control for advertising reports. When it is enabled,
+        advertising reports will be discarded by the controller if the number of unprocessed advertising
+        reports exceeds the size of BLE adv report flow control.
+
+config BLE_ADV_REPORT_FLOW_CONTROL_NUM
+    int "BLE adv report flow control number"
+    depends on BLE_ADV_REPORT_FLOW_CONTROL_SUPPORTED
+    range 50 1000
+    default 100
+    help
+        The number of unprocessed advertising report that Bluedroid can save.If you set
+        `BLE_ADV_REPORT_FLOW_CONTROL_NUM` to a small value, this may cause adv packets lost.
+        If you set `BLE_ADV_REPORT_FLOW_CONTROL_NUM` to a large value, Bluedroid may cache a
+        lot of adv packets and this may cause system memory run out. For example, if you set
+        it to 50, the maximum memory consumed by host is 35 * 50 bytes. Please set
+        `BLE_ADV_REPORT_FLOW_CONTROL_NUM` according to your system free memory and handle adv
+        packets as fast as possible, otherwise it will cause adv packets lost.
+
+config BLE_ADV_REPORT_DISCARD_THRSHOLD
+    int "BLE adv lost event threshold value"
+    depends on BLE_ADV_REPORT_FLOW_CONTROL_SUPPORTED
+    range 1 1000
+    default 20
+    help
+        When adv report flow control is enabled, The ADV lost event will be generated when the number
+        of ADV packets lost in the controller reaches this threshold. It is better to set a larger value.
+        If you set `BLE_ADV_REPORT_DISCARD_THRSHOLD` to a small value or printf every adv lost event, it
+        may cause adv packets lost more.
 
 
 endmenu

components/bt/bluedroid/api/esp_gap_ble_api.c

@@ -449,6 +449,69 @@ esp_err_t esp_ble_gap_config_scan_rsp_data_raw(uint8_t *raw_data, uint32_t raw_d
 
 }
 
+esp_err_t esp_ble_gap_add_duplicate_scan_exceptional_device(esp_ble_duplicate_exceptional_info_type_t type, esp_duplicate_info_t device_info)
+{
+    btc_msg_t msg;
+    btc_ble_gap_args_t arg;
+
+    if (esp_bluedroid_get_status() != ESP_BLUEDROID_STATUS_ENABLED) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    if (!device_info){
+        return ESP_ERR_INVALID_SIZE;
+    }
+    msg.sig = BTC_SIG_API_CALL;
+    msg.pid = BTC_PID_GAP_BLE;
+    msg.act = BTC_GAP_BLE_UPDATE_DUPLICATE_SCAN_EXCEPTIONAL_LIST;
+    arg.update_duplicate_exceptional_list.subcode = ESP_BLE_DUPLICATE_EXCEPTIONAL_LIST_ADD;
+    arg.update_duplicate_exceptional_list.info_type = type;
+    memcpy(arg.update_duplicate_exceptional_list.device_info, device_info, sizeof(esp_bd_addr_t));
+
+    return (btc_transfer_context(&msg, &arg, sizeof(btc_ble_gap_args_t), NULL)
+                == BT_STATUS_SUCCESS ? ESP_OK : ESP_FAIL);
+}
+
+esp_err_t esp_ble_gap_remove_duplicate_scan_exceptional_device(esp_ble_duplicate_exceptional_info_type_t type, esp_duplicate_info_t device_info)
+{
+    btc_msg_t msg;
+    btc_ble_gap_args_t arg;
+
+    if (esp_bluedroid_get_status() != ESP_BLUEDROID_STATUS_ENABLED) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    if (!device_info){
+        return ESP_ERR_INVALID_SIZE;
+    }
+    msg.sig = BTC_SIG_API_CALL;
+    msg.pid = BTC_PID_GAP_BLE;
+    msg.act = BTC_GAP_BLE_UPDATE_DUPLICATE_SCAN_EXCEPTIONAL_LIST;
+    arg.update_duplicate_exceptional_list.subcode = ESP_BLE_DUPLICATE_EXCEPTIONAL_LIST_REMOVE;
+    arg.update_duplicate_exceptional_list.info_type = type;
+    memcpy(arg.update_duplicate_exceptional_list.device_info, device_info, sizeof(esp_bd_addr_t));
+
+    return (btc_transfer_context(&msg, &arg, sizeof(btc_ble_gap_args_t), NULL)
+                == BT_STATUS_SUCCESS ? ESP_OK : ESP_FAIL);
+}
+
+esp_err_t esp_ble_gap_clean_duplicate_scan_exceptional_list(esp_duplicate_scan_exceptional_list_type_t list_type)
+{
+    btc_msg_t msg;
+    btc_ble_gap_args_t arg;
+
+    if (esp_bluedroid_get_status() != ESP_BLUEDROID_STATUS_ENABLED) {
+        return ESP_ERR_INVALID_STATE;
+    }
+
+    msg.sig = BTC_SIG_API_CALL;
+    msg.pid = BTC_PID_GAP_BLE;
+    msg.act = BTC_GAP_BLE_UPDATE_DUPLICATE_SCAN_EXCEPTIONAL_LIST;
+    arg.update_duplicate_exceptional_list.subcode = ESP_BLE_DUPLICATE_EXCEPTIONAL_LIST_CLEAN;
+    arg.update_duplicate_exceptional_list.info_type = list_type;
+
+    return (btc_transfer_context(&msg, &arg, sizeof(btc_ble_gap_args_t), NULL)
+                == BT_STATUS_SUCCESS ? ESP_OK : ESP_FAIL);
+}
+
 #if (SMP_INCLUDED == TRUE)
 esp_err_t esp_ble_gap_set_security_param(esp_ble_sm_param_t param_type,
         void *value, uint8_t len)

components/bt/bluedroid/api/include/api/esp_gap_ble_api.h

@@ -161,6 +161,7 @@ typedef enum {
     ESP_GAP_BLE_GET_BOND_DEV_COMPLETE_EVT,                  /*!< When get the bond device list complete, the event comes */
     ESP_GAP_BLE_READ_RSSI_COMPLETE_EVT,                     /*!< When read the rssi complete, the event comes */
     ESP_GAP_BLE_UPDATE_WHITELIST_COMPLETE_EVT,              /*!< When add or remove whitelist complete, the event comes */
+    ESP_GAP_BLE_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_COMPLETE_EVT,  /*!< When update duplicate exceptional list complete, the event comes */
     ESP_GAP_BLE_EVT_MAX,
 } esp_gap_ble_cb_event_t;
 /// This is the old name, just for backwards compatibility
@@ -552,6 +553,7 @@ typedef enum {
     ESP_GAP_SEARCH_DISC_CMPL_EVT           = 4,      /*!< Discovery complete. */
     ESP_GAP_SEARCH_DI_DISC_CMPL_EVT        = 5,      /*!< Discovery complete. */
     ESP_GAP_SEARCH_SEARCH_CANCEL_CMPL_EVT  = 6,      /*!< Search cancelled */
+    ESP_GAP_SEARCH_INQ_DISCARD_NUM_EVT     = 7,      /*!< The number of pkt discarded by flow control */
 } esp_gap_search_evt_t;
 
 /**
@@ -570,6 +572,28 @@ typedef enum{
     ESP_BLE_WHITELIST_REMOVE     = 0X00,    /*!< remove mac from whitelist */
     ESP_BLE_WHITELIST_ADD        = 0X01,    /*!< add address to whitelist */
 }esp_ble_wl_opration_t;
+
+typedef enum {
+    ESP_BLE_DUPLICATE_EXCEPTIONAL_LIST_ADD      = 0,  /*!< Add device info into duplicate scan exceptional list */
+    ESP_BLE_DUPLICATE_EXCEPTIONAL_LIST_REMOVE,        /*!< Remove device info from duplicate scan exceptional list */
+    ESP_BLE_DUPLICATE_EXCEPTIONAL_LIST_CLEAN,         /*!< Clean duplicate scan exceptional list */
+} esp_bt_duplicate_exceptional_subcode_type_t;
+
+#define BLE_BIT(n) (1UL<<(n))
+
+typedef enum {
+    ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_INFO_ADV_ADDR       = 0,  /*!< BLE advertising address , device info will be added into ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_ADDR_LIST */
+    ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_INFO_MESH_LINK_ID,        /*!< BLE mesh link ID, it is for BLE mesh, device info will be added into ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_MESH_LINK_ID_LIST */
+} esp_ble_duplicate_exceptional_info_type_t;
+
+typedef enum {
+    ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_ADDR_LIST         = BLE_BIT(0), /*!< duplicate scan exceptional addr list */
+    ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_MESH_LINK_ID_LIST = BLE_BIT(1), /*!< duplicate scan exceptional mesh link ID list */
+    ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_ALL_LIST = (BLE_BIT(0) | BLE_BIT(1)), /*!< duplicate scan exceptional all list */
+} esp_duplicate_scan_exceptional_list_type_t;
+
+typedef uint8_t esp_duplicate_info_t[ESP_BD_ADDR_LEN];
+
 /**
  * @brief Gap callback parameters union
  */
@@ -607,6 +631,7 @@ typedef union {
         int num_resps;                              /*!< Scan result number */
         uint8_t adv_data_len;                       /*!< Adv data length */
         uint8_t scan_rsp_len;                       /*!< Scan response length */
+        uint32_t num_dis;                          /*!< The number of discard packets */
     } scan_rst;                                     /*!< Event parameter of ESP_GAP_BLE_SCAN_RESULT_EVT */
     /**
      * @brief ESP_GAP_BLE_ADV_DATA_RAW_SET_COMPLETE_EVT
@@ -715,6 +740,15 @@ typedef union {
         esp_bt_status_t status;                     /*!< Indicate the add or remove whitelist operation success status */
         esp_ble_wl_opration_t wl_opration;          /*!< The value is ESP_BLE_WHITELIST_ADD if add address to whitelist operation success, ESP_BLE_WHITELIST_REMOVE if remove address from the whitelist operation success */
     } update_whitelist_cmpl;                        /*!< Event parameter of ESP_GAP_BLE_UPDATE_WHITELIST_COMPLETE_EVT */
+    /**
+     * @brief ESP_GAP_BLE_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_COMPLETE_EVT
+     */
+    struct ble_update_duplicate_exceptional_list_cmpl_evt_param {
+        esp_bt_status_t status;                     /*!< Indicate update duplicate scan exceptional list operation success status */
+        uint8_t         subcode;                    /*!< Define in esp_bt_duplicate_exceptional_subcode_type_t */
+        uint16_t         length;                     /*!< The length of device_info */
+        esp_duplicate_info_t device_info;           /*!< device information, when subcode is ESP_BLE_DUPLICATE_EXCEPTIONAL_LIST_CLEAN, the value is invalid */
+    } update_duplicate_exceptional_list_cmpl;       /*!< Event parameter of ESP_GAP_BLE_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_COMPLETE_EVT */
 } esp_ble_gap_cb_param_t;
 
 /**
@@ -1007,6 +1041,43 @@ esp_err_t esp_ble_gap_config_scan_rsp_data_raw(uint8_t *raw_data, uint32_t raw_d
  */
 esp_err_t esp_ble_gap_read_rssi(esp_bd_addr_t remote_addr);
 
+/**
+ * @brief           This function is called to add a device info into the duplicate scan exceptional list.
+ *
+ *
+ * @param[in]       type: device info type, it is defined in esp_ble_duplicate_exceptional_info_type_t
+ * @param[in]       device_info: the device information.
+ * @return
+ *                  - ESP_OK : success
+ *                  - other  : failed
+ */
+esp_err_t esp_ble_gap_add_duplicate_scan_exceptional_device(esp_ble_duplicate_exceptional_info_type_t type, esp_duplicate_info_t device_info);
+
+/**
+ * @brief           This function is called to remove a device info from the duplicate scan exceptional list.
+ *
+ *
+ * @param[in]       type: device info type, it is defined in esp_ble_duplicate_exceptional_info_type_t
+ * @param[in]       device_info: the device information.
+ * @return
+ *                  - ESP_OK : success
+ *                  - other  : failed
+ */
+esp_err_t esp_ble_gap_remove_duplicate_scan_exceptional_device(esp_ble_duplicate_exceptional_info_type_t type, esp_duplicate_info_t device_info);
+
+/**
+ * @brief           This function is called to clean the duplicate scan exceptional list.
+ *                  This API will delete all device information in the duplicate scan exceptional list.
+ *
+ *
+ * @param[in]       list_type: duplicate scan exceptional list type, the value can be one or more of esp_duplicate_scan_exceptional_list_type_t.
+ *
+ * @return
+ *                  - ESP_OK : success
+ *                  - other  : failed
+ */
+esp_err_t esp_ble_gap_clean_duplicate_scan_exceptional_list(esp_duplicate_scan_exceptional_list_type_t list_type);
+
 #if (SMP_INCLUDED == TRUE)
 /**
 * @brief             Set a GAP security parameter value. Overrides the default value.

components/bt/bluedroid/bta/dm/bta_dm_act.c

@@ -127,6 +127,7 @@ static void bta_dm_remove_sec_dev_entry(BD_ADDR remote_bd_addr);
 #endif  ///SMP_INCLUDED == TRUE
 static void bta_dm_observe_results_cb(tBTM_INQ_RESULTS *p_inq, UINT8 *p_eir);
 static void bta_dm_observe_cmpl_cb(void *p_result);
+static void bta_dm_observe_discard_cb (uint32_t num_dis);
 static void bta_dm_delay_role_switch_cback(TIMER_LIST_ENT *p_tle);
 extern void sdpu_uuid16_to_uuid128(UINT16 uuid16, UINT8 *p_uuid128);
 static void bta_dm_disable_timer_cback(TIMER_LIST_ENT *p_tle);
@@ -4258,6 +4259,28 @@ static void bta_dm_observe_cmpl_cb (void *p_result)
     }
 }
 
+/*******************************************************************************
+**
+** Function         bta_dm_observe_discard_cb
+**
+** Description      Callback for BLE Observe lost
+**
+**
+** Returns          void
+**
+*******************************************************************************/
+static void bta_dm_observe_discard_cb (uint32_t num_dis)
+{
+    tBTA_DM_SEARCH  data;
+
+    APPL_TRACE_DEBUG("bta_dm_observe_discard_cb");
+
+    data.inq_dis.num_dis = num_dis;
+    if (bta_dm_search_cb.p_scan_cback) {
+        bta_dm_search_cb.p_scan_cback(BTA_DM_INQ_DISCARD_NUM_EVT, &data);
+    }
+}
+
 #if (SMP_INCLUDED == TRUE)
 /*******************************************************************************
 **
@@ -4789,7 +4812,7 @@ void bta_dm_ble_scan (tBTA_DM_MSG *p_data)
         bta_dm_search_cb.p_scan_cback = p_data->ble_scan.p_cback;
 
         if ((status = BTM_BleScan(TRUE, p_data->ble_scan.duration,
-                                     bta_dm_observe_results_cb, bta_dm_observe_cmpl_cb)) != BTM_CMD_STARTED) {
+                                     bta_dm_observe_results_cb, bta_dm_observe_cmpl_cb, bta_dm_observe_discard_cb)) != BTM_CMD_STARTED) {
             APPL_TRACE_WARNING(" %s start scan failed. status=0x%x\n", __FUNCTION__, status);
         }
 
@@ -4799,7 +4822,7 @@ void bta_dm_ble_scan (tBTA_DM_MSG *p_data)
         }
     } else {
         bta_dm_search_cb.p_scan_cback = NULL;
-        status = BTM_BleScan(FALSE, 0, NULL, NULL);
+        status = BTM_BleScan(FALSE, 0, NULL, NULL, NULL);
 
         if (status != BTM_CMD_STARTED){
             APPL_TRACE_WARNING(" %s stop scan failed, status=0x%x\n", __FUNCTION__, status);
@@ -4856,6 +4879,22 @@ void bta_dm_ble_set_adv_params_all  (tBTA_DM_MSG *p_data)
     }
 }
 
+/*******************************************************************************
+**
+** Function         bta_dm_ble_update_duplicate_exceptional_list
+**
+** Description      This function is to update duplicate scan exceptional list
+**
+**
+*******************************************************************************/
+void bta_dm_ble_update_duplicate_exceptional_list(tBTA_DM_MSG *p_data)
+{
+    BTM_UpdateBleDuplicateExceptionalList(p_data->ble_duplicate_exceptional_list.subcode,
+                                          p_data->ble_duplicate_exceptional_list.type,
+                                          p_data->ble_duplicate_exceptional_list.device_info,
+                                          p_data->ble_duplicate_exceptional_list.exceptional_list_cb);
+}
+
 /*******************************************************************************
 **
 ** Function         bta_dm_ble_set_adv_config

components/bt/bluedroid/bta/dm/bta_dm_api.c

@@ -1258,6 +1258,34 @@ void BTA_DmBleSetScanRspRaw (UINT8 *p_raw_scan_rsp, UINT32 raw_scan_rsp_len,
     }
 }
 
+/*******************************************************************************
+**
+** Function         BTA_DmUpdateDuplicateExceptionalList
+**
+** Description      This function is called to update duplicate scan exceptional list
+**
+** Parameters       subcode : add, remove or clean duplicate scan exceptional list.
+**                  type : device info type.
+**                  device_info:  device info
+**                  p_update_duplicate_ignore_list_cback :  update complete callback.
+**
+** Returns          None
+**
+*******************************************************************************/
+void BTA_DmUpdateDuplicateExceptionalList(UINT8 subcode, UINT32 type, BD_ADDR device_info, tBTA_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_CMPL_CBACK p_update_duplicate_exceptional_list_cback)
+{
+    tBTA_DM_API_UPDATE_DUPLICATE_EXCEPTIONAL_LIST *p_msg;
+    if ((p_msg = (tBTA_DM_API_UPDATE_DUPLICATE_EXCEPTIONAL_LIST *)osi_malloc(sizeof(tBTA_DM_API_UPDATE_DUPLICATE_EXCEPTIONAL_LIST))) != NULL) {
+        p_msg->hdr.event = BTA_DM_API_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_EVT;
+        p_msg->subcode = subcode;
+        p_msg->type = type;
+        p_msg->exceptional_list_cb = p_update_duplicate_exceptional_list_cback;
+        memcpy(p_msg->device_info, device_info, sizeof(BD_ADDR));
+
+        bta_sys_sendmsg(p_msg);
+    }
+}
+
 /*******************************************************************************
 **
 ** Function         BTA_DmBleSetStorageParams

components/bt/bluedroid/bta/dm/bta_dm_main.c

@@ -157,6 +157,7 @@ const tBTA_DM_ACTION bta_dm_action[BTA_DM_MAX_EVT] = {
     bta_dm_update_white_list,               /* BTA_DM_API_UPDATE_WHITE_LIST_EVT */
     bta_dm_ble_read_adv_tx_power,           /* BTA_DM_API_BLE_READ_ADV_TX_POWER_EVT */
     bta_dm_ble_read_rssi,                   /* BTA_DM_API_BLE_READ_RSSI_EVT */
+    bta_dm_ble_update_duplicate_exceptional_list,/* BTA_DM_API_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_EVT */
 };
 
 

components/bt/bluedroid/bta/dm/include/bta_dm_int.h

@@ -155,6 +155,7 @@ enum {
     BTA_DM_API_UPDATE_WHITE_LIST_EVT,
     BTA_DM_API_BLE_READ_ADV_TX_POWER_EVT,
     BTA_DM_API_BLE_READ_RSSI_EVT,
+    BTA_DM_API_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_EVT,
     BTA_DM_MAX_EVT
 };
 
@@ -193,6 +194,14 @@ typedef struct {
     tBTA_ADD_WHITELIST_CBACK *add_wl_cb;
 }tBTA_DM_API_UPDATE_WHITE_LIST;
 
+typedef struct {
+    BT_HDR    hdr;
+    UINT8     subcode;
+    UINT32    type;
+    BD_ADDR   device_info;
+    tBTA_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_CMPL_CBACK *exceptional_list_cb;
+}tBTA_DM_API_UPDATE_DUPLICATE_EXCEPTIONAL_LIST;
+
 typedef struct {
     BT_HDR       hdr;
     tBTA_CMPL_CB *read_tx_power_cb;
@@ -848,6 +857,7 @@ typedef union {
     tBTA_DM_API_TRACK_ADVERTISER        ble_track_advert;
     tBTA_DM_API_ENERGY_INFO             ble_energy_info;
     tBTA_DM_API_BLE_DISCONNECT          ble_disconnect;
+    tBTA_DM_API_UPDATE_DUPLICATE_EXCEPTIONAL_LIST ble_duplicate_exceptional_list;
 #endif
 
     tBTA_DM_API_REMOVE_ACL              remove_acl;
@@ -1243,7 +1253,7 @@ extern void bta_dm_ble_set_scan_rsp (tBTA_DM_MSG *p_data);
 extern void bta_dm_ble_set_scan_rsp_raw (tBTA_DM_MSG *p_data);
 extern void bta_dm_ble_broadcast (tBTA_DM_MSG *p_data);
 extern void bta_dm_ble_set_data_length(tBTA_DM_MSG *p_data);
-
+extern void bta_dm_ble_update_duplicate_exceptional_list(tBTA_DM_MSG *p_data);
 #if BLE_ANDROID_CONTROLLER_SCAN_FILTER == TRUE
 extern void bta_dm_cfg_filter_cond (tBTA_DM_MSG *p_data);
 extern void bta_dm_scan_filter_param_setup (tBTA_DM_MSG *p_data);

components/bt/bluedroid/bta/include/bta/bta_api.h

@@ -399,6 +399,8 @@ typedef struct {
     UINT8                   tx_power;
 } tBTA_BLE_ADV_DATA;
 
+typedef void (tBTA_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_CMPL_CBACK) (tBTA_STATUS status, uint8_t subcode, uint32_t length, uint8_t *device_info);
+
 typedef void (tBTA_SET_ADV_DATA_CMPL_CBACK) (tBTA_STATUS status);
 
 typedef tBTM_START_ADV_CMPL_CBACK tBTA_START_ADV_CMPL_CBACK;
@@ -1022,6 +1024,7 @@ typedef struct {
 #define BTA_DM_DISC_CMPL_EVT            4       /* Discovery complete. */
 #define BTA_DM_DI_DISC_CMPL_EVT         5       /* Discovery complete. */
 #define BTA_DM_SEARCH_CANCEL_CMPL_EVT   6       /* Search cancelled */
+#define BTA_DM_INQ_DISCARD_NUM_EVT      7       /* The number of inquiry discarded packets */
 
 typedef UINT8 tBTA_DM_SEARCH_EVT;
 
@@ -1053,6 +1056,11 @@ typedef struct {
     UINT8           num_resps;          /* Number of inquiry responses. */
 } tBTA_DM_INQ_CMPL;
 
+/* Structure associated with BTA_DM_INQ_DISCARD_NUM_EVT */
+typedef struct {
+    UINT32           num_dis;          /* The number of inquiry discarded packets. */
+} tBTA_DM_INQ_DISCARD;
+
 /* Structure associated with BTA_DM_DI_DISC_CMPL_EVT */
 typedef struct {
     BD_ADDR             bd_addr;        /* BD address peer device. */
@@ -1090,6 +1098,7 @@ typedef union {
     tBTA_DM_DISC_RES    disc_res;       /* Discovery result for a peer device. */
     tBTA_DM_DISC_BLE_RES    disc_ble_res;   /* Discovery result for GATT based service */
     tBTA_DM_DI_DISC_CMPL    di_disc;        /* DI discovery result for a peer device */
+    tBTA_DM_INQ_DISCARD     inq_dis;       /* the discarded packets information of inquiry */
 } tBTA_DM_SEARCH;
 
 /* Structure of search callback event and structures */
@@ -2237,6 +2246,24 @@ extern void BTA_DmBleSetScanRsp (tBTA_BLE_AD_MASK data_mask,
 extern void BTA_DmBleSetScanRspRaw (UINT8 *p_raw_scan_rsp, UINT32 raw_scan_rsp_len,
                                     tBTA_SET_ADV_DATA_CMPL_CBACK *p_scan_rsp_data_cback);
 
+/*******************************************************************************
+**
+** Function         BTA_DmUpdateDuplicateExceptionalList
+**
+** Description      This function is called to update duplicate scan exceptional list
+**
+** Parameters       subcode : add, remove or clean duplicate scan exceptional list.
+**                  type : device info type.
+**                  device_info:  device info
+**                  p_update_duplicate_ignore_list_cback :  update complete callback.
+**
+** Returns          None
+**
+*******************************************************************************/
+extern void BTA_DmUpdateDuplicateExceptionalList(UINT8 subcode, UINT32 type, 
+                                                BD_ADDR device_info, 
+                                                tBTA_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_CMPL_CBACK p_update_duplicate_exceptional_list_cback);
+
 /*******************************************************************************
 **
 ** Function         BTA_DmBleBroadcast

components/bt/bluedroid/btc/profile/std/gap/btc_gap_ble.c

@@ -426,6 +426,35 @@ static void btc_stop_adv_callback(uint8_t status)
     }
 }
 
+void btc_update_duplicate_exceptional_list_callback(tBTA_STATUS status, uint8_t subcode, uint32_t length, uint8_t *device_info)
+{
+    esp_ble_gap_cb_param_t param;
+    bt_status_t ret;
+    btc_msg_t msg;
+
+    msg.sig = BTC_SIG_API_CB;
+    msg.pid = BTC_PID_GAP_BLE;
+    msg.act = ESP_GAP_BLE_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_COMPLETE_EVT;
+    param.update_duplicate_exceptional_list_cmpl.status = status;
+    param.update_duplicate_exceptional_list_cmpl.subcode = subcode;
+    if(length > sizeof(param.update_duplicate_exceptional_list_cmpl.device_info)) {
+        length = sizeof(param.update_duplicate_exceptional_list_cmpl.device_info);
+    }
+    param.update_duplicate_exceptional_list_cmpl.length = length;
+    memcpy(param.update_duplicate_exceptional_list_cmpl.device_info, device_info, length);
+    ret = btc_transfer_context(&msg, &param, sizeof(esp_ble_gap_cb_param_t), NULL);
+
+    if (ret != BT_STATUS_SUCCESS) {
+        BTC_TRACE_ERROR("%s btc_transfer_context failed\n", __func__);
+    }
+}
+
+static void btc_ble_update_duplicate_exceptional_list(uint8_t subcode, uint32_t info_type, BD_ADDR device_info,
+                                 tBTA_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_CMPL_CBACK p_update_duplicate_ignore_list_cback)
+{
+    BTA_DmUpdateDuplicateExceptionalList(subcode, info_type, device_info, p_update_duplicate_ignore_list_cback);
+}
+
 static void btc_ble_start_advertising (esp_ble_adv_params_t *ble_adv_params, tBTA_START_ADV_CMPL_CBACK start_adv_cback)
 {
     tBLE_BD_ADDR peer_addr;
@@ -575,6 +604,9 @@ static void btc_search_callback(tBTA_DM_SEARCH_EVT event, tBTA_DM_SEARCH *p_data
     case BTA_DM_SEARCH_CANCEL_CMPL_EVT:
         BTC_TRACE_DEBUG("BTA_DM_SEARCH_CANCEL_CMPL_EVT\n");
         break;
+    case BTA_DM_INQ_DISCARD_NUM_EVT:
+        param.scan_rst.num_dis = p_data->inq_dis.num_dis;
+        break;
     default:
         BTC_TRACE_ERROR("%s : Unknown event 0x%x\n", __FUNCTION__, event);
         return;
@@ -1086,6 +1118,12 @@ void btc_gap_ble_call_handler(btc_msg_t *msg)
                                       arg->cfg_scan_rsp_data_raw.raw_scan_rsp_len,
                                       btc_scan_rsp_data_raw_callback);
         break;
+    case BTC_GAP_BLE_UPDATE_DUPLICATE_SCAN_EXCEPTIONAL_LIST:
+        btc_ble_update_duplicate_exceptional_list(arg->update_duplicate_exceptional_list.subcode,
+                                                arg->update_duplicate_exceptional_list.info_type,
+                                                arg->update_duplicate_exceptional_list.device_info,
+                                                btc_update_duplicate_exceptional_list_callback);
+        break;
 #if (SMP_INCLUDED == TRUE)
     case BTC_GAP_BLE_SET_ENCRYPTION_EVT: {
         BD_ADDR bd_addr;

components/bt/bluedroid/btc/profile/std/include/btc_gap_ble.h

@@ -46,6 +46,7 @@ typedef enum {
     BTC_GAP_BLE_CONFIRM_REPLY_EVT,
     BTC_GAP_BLE_DISCONNECT_EVT,
     BTC_GAP_BLE_REMOVE_BOND_DEV_EVT,
+    BTC_GAP_BLE_UPDATE_DUPLICATE_SCAN_EXCEPTIONAL_LIST,
 } btc_gap_ble_act_t;
 
 /* btc_ble_gap_args_t */
@@ -94,6 +95,12 @@ typedef union {
         bool add_remove;
         esp_bd_addr_t remote_bda;
     }update_white_list;
+    //BTC_GAP_BLE_UPDATE_DUPLICATE_SCAN_EXCEPTIONAL_LIST
+    struct update_duplicate_exceptional_list_args {
+        uint8_t  subcode;
+        uint32_t  info_type;
+        esp_duplicate_info_t device_info;
+    }update_duplicate_exceptional_list;
     //BTC_GAP_BLE_ACT_SET_CONN_PARAMS
     struct set_conn_params_args {
         esp_bd_addr_t bd_addr;

components/bt/bluedroid/common/include/common/bt_target.h

@@ -130,6 +130,24 @@
 #define BLE_PRIVACY_SPT           FALSE
 #endif  /* CONFIG_SMP_ENABLE */
 
+#ifndef CONFIG_BLE_ADV_REPORT_FLOW_CONTROL_SUPPORTED
+#define BLE_ADV_REPORT_FLOW_CONTROL  FALSE
+#else
+#define BLE_ADV_REPORT_FLOW_CONTROL  CONFIG_BLE_ADV_REPORT_FLOW_CONTROL_SUPPORTED
+#endif /* CONFIG_BLE_ADV_REPORT_FLOW_CONTROL_SUPPORTED */
+
+#ifndef CONFIG_BLE_ADV_REPORT_FLOW_CONTROL_NUM
+#define BLE_ADV_REPORT_FLOW_CONTROL_NUM   100
+#else 
+#define BLE_ADV_REPORT_FLOW_CONTROL_NUM     CONFIG_BLE_ADV_REPORT_FLOW_CONTROL_NUM
+#endif /* CONFIG_BLE_ADV_REPORT_FLOW_CONTROL_NUM */
+
+#ifndef CONFIG_BLE_ADV_REPORT_DISCARD_THRSHOLD
+#define BLE_ADV_REPORT_DISCARD_THRSHOLD  20
+#else 
+#define BLE_ADV_REPORT_DISCARD_THRSHOLD  CONFIG_BLE_ADV_REPORT_DISCARD_THRSHOLD
+#endif /* CONFIG_BLE_ADV_REPORT_DISCARD_THRSHOLD */
+
 #if (CONFIG_BT_ACL_CONNECTIONS)
 #define MAX_ACL_CONNECTIONS  CONFIG_BT_ACL_CONNECTIONS
 #define GATT_MAX_PHY_CHANNEL CONFIG_BT_ACL_CONNECTIONS

components/bt/bluedroid/device/controller.c

@@ -100,7 +100,11 @@ static void start_up(void)
     response = AWAIT_COMMAND(packet_factory->make_set_c2h_flow_control(HCI_HOST_FLOW_CTRL_ACL_ON));
     packet_parser->parse_generic_command_complete(response);
 #endif ///C2H_FLOW_CONTROL_INCLUDED == TRUE
-
+#if (BLE_ADV_REPORT_FLOW_CONTROL == TRUE)
+    // Enable adv flow control
+    response = AWAIT_COMMAND(packet_factory->make_set_adv_report_flow_control(HCI_HOST_FLOW_CTRL_ADV_REPORT_ON, (uint16_t)BLE_ADV_REPORT_FLOW_CONTROL_NUM, (uint16_t)BLE_ADV_REPORT_DISCARD_THRSHOLD));
+    packet_parser->parse_generic_command_complete(response);
+#endif
     // Tell the controller about our buffer sizes and buffer counts next
     // TODO(zachoverflow): factor this out. eww l2cap contamination. And why just a hardcoded 10?
     response = AWAIT_COMMAND(

components/bt/bluedroid/hci/hci_hal_h4.c

@@ -55,6 +55,7 @@ static const uint16_t outbound_event_types[] = {
 typedef struct {
     size_t buffer_size;
     fixed_queue_t *rx_q;
+    uint16_t adv_free_num;
 } hci_hal_env_t;
 
 
@@ -81,6 +82,7 @@ static void hci_hal_env_init(
     assert(max_buffer_count > 0);
 
     hci_hal_env.buffer_size = buffer_size;
+    hci_hal_env.adv_free_num = 0;
 
     hci_hal_env.rx_q = fixed_queue_new(max_buffer_count);
     if (hci_hal_env.rx_q) {
@@ -102,8 +104,11 @@ static bool hal_open(const hci_hal_callbacks_t *upper_callbacks)
 {
     assert(upper_callbacks != NULL);
     callbacks = upper_callbacks;
-
+#if (BLE_ADV_REPORT_FLOW_CONTROL == TRUE)
+    hci_hal_env_init(HCI_HAL_SERIAL_BUFFER_SIZE, BLE_ADV_REPORT_FLOW_CONTROL_NUM + L2CAP_HOST_FC_ACL_BUFS + QUEUE_SIZE_MAX); // adv flow control num + ACL flow control num + hci cmd numeber
+#else
     hci_hal_env_init(HCI_HAL_SERIAL_BUFFER_SIZE, QUEUE_SIZE_MAX);
+#endif
 
     xHciH4Queue = xQueueCreate(HCI_H4_QUEUE_LEN, sizeof(BtTaskEvt_t));
     xTaskCreatePinnedToCore(hci_hal_h4_rx_handler, HCI_H4_TASK_NAME, HCI_H4_TASK_STACK_SIZE, NULL, HCI_H4_TASK_PRIO, &xHciH4TaskHandle, HCI_H4_TASK_PINNED_TO_CORE);
@@ -223,12 +228,37 @@ static void hci_packet_complete(BT_HDR *packet){
 bool host_recv_adv_packet(BT_HDR *packet)
 {
     assert(packet);
-    if(packet->data[0] == DATA_TYPE_EVENT && packet->data[1] == HCI_BLE_EVENT && packet->data[3] ==  HCI_BLE_ADV_PKT_RPT_EVT) {
-        return true;
+    if(packet->data[0] == DATA_TYPE_EVENT && packet->data[1] == HCI_BLE_EVENT) {
+        if(packet->data[3] ==  HCI_BLE_ADV_PKT_RPT_EVT 
+#if (BLE_ADV_REPORT_FLOW_CONTROL == TRUE)
+        || packet->data[3] ==  HCI_BLE_ADV_DISCARD_REPORT_EVT
+#endif
+        ) {
+            return true;
+        }
     }
     return false;
 }
 
+#if (BLE_ADV_REPORT_FLOW_CONTROL == TRUE)
+static void hci_update_adv_report_flow_control(BT_HDR *packet)
+{
+    // this is adv packet
+    if(host_recv_adv_packet(packet)) {
+        // update adv free number
+        hci_hal_env.adv_free_num ++;
+        if (esp_vhci_host_check_send_available()){
+            // send hci cmd
+            btsnd_hcic_ble_update_adv_report_flow_control(hci_hal_env.adv_free_num);
+            hci_hal_env.adv_free_num = 0;
+        } else {
+            //do nothing
+        }
+    }
+
+}
+#endif
+
 static void hci_hal_h4_hdl_rx_packet(BT_HDR *packet)
 {
     uint8_t type, hdr_size;
@@ -282,6 +312,11 @@ static void hci_hal_h4_hdl_rx_packet(BT_HDR *packet)
         osi_free(packet);
         return;
     }
+
+#if (BLE_ADV_REPORT_FLOW_CONTROL == TRUE)
+    hci_update_adv_report_flow_control(packet);
+#endif
+
 #if SCAN_QUEUE_CONGEST_CHECK
     if(BTU_check_queue_is_congest() && host_recv_adv_packet(packet)) {
         HCI_TRACE_ERROR("BtuQueue is congested");
@@ -289,7 +324,6 @@ static void hci_hal_h4_hdl_rx_packet(BT_HDR *packet)
         return;
     }
 #endif
-
     packet->event = outbound_event_types[PACKET_TYPE_TO_INDEX(type)];
     callbacks->packet_ready(packet);
 }

components/bt/bluedroid/hci/hci_layer.c

@@ -317,7 +317,11 @@ static void event_command_ready(fixed_queue_t *queue)
 
     wait_entry = fixed_queue_dequeue(queue);
 
-    if(wait_entry->opcode == HCI_HOST_NUM_PACKETS_DONE){
+    if(wait_entry->opcode == HCI_HOST_NUM_PACKETS_DONE 
+#if (BLE_ADV_REPORT_FLOW_CONTROL == TRUE)
+    || wait_entry->opcode == HCI_VENDOR_BLE_ADV_REPORT_FLOW_CONTROL
+#endif
+    ){
         packet_fragmenter->fragment_and_dispatch(wait_entry->command);
         osi_free(wait_entry->command);
         osi_free(wait_entry);

components/bt/bluedroid/hci/hci_packet_factory.c

@@ -53,6 +53,18 @@ static BT_HDR *make_set_c2h_flow_control(uint8_t enable)
     return packet;
 }
 
+static BT_HDR *make_set_adv_report_flow_control(uint8_t enable, uint16_t num, uint16_t lost_threshold)
+{
+    uint8_t *stream;
+    const uint8_t parameter_size = 1 + 2 + 2;
+    BT_HDR *packet = make_command(HCI_VENDOR_BLE_SET_ADV_FLOW_CONTROL, parameter_size, &stream);
+
+    UINT8_TO_STREAM(stream, enable);
+    UINT16_TO_STREAM(stream, num);
+    UINT16_TO_STREAM(stream, lost_threshold);
+    return packet;
+}
+
 static BT_HDR *make_host_buffer_size(uint16_t acl_size, uint8_t sco_size, uint16_t acl_count, uint16_t sco_count)
 {
     uint8_t *stream;
@@ -239,6 +251,7 @@ static const hci_packet_factory_t interface = {
     make_reset,
     make_read_buffer_size,
     make_set_c2h_flow_control,
+    make_set_adv_report_flow_control,
     make_host_buffer_size,
     make_read_local_version_info,
     make_read_bd_addr,

components/bt/bluedroid/hci/include/hci/hci_packet_factory.h

@@ -26,6 +26,7 @@ typedef struct {
     BT_HDR *(*make_reset)(void);
     BT_HDR *(*make_read_buffer_size)(void);
     BT_HDR *(*make_set_c2h_flow_control)(uint8_t enable);
+    BT_HDR *(*make_set_adv_report_flow_control)(uint8_t enable, uint16_t num, uint16_t lost_threshold);
     BT_HDR *(*make_host_buffer_size)(uint16_t acl_size, uint8_t sco_size, uint16_t acl_count, uint16_t sco_count);
     BT_HDR *(*make_read_local_version_info)(void);
     BT_HDR *(*make_read_bd_addr)(void);

components/bt/bluedroid/stack/btm/btm_ble_gap.c

@@ -439,7 +439,7 @@ tBTM_STATUS BTM_BleObserve(BOOLEAN start, UINT32 duration,
 **
 *******************************************************************************/
 tBTM_STATUS BTM_BleScan(BOOLEAN start, UINT32 duration,
-                           tBTM_INQ_RESULTS_CB *p_results_cb, tBTM_CMPL_CB *p_cmpl_cb)
+                           tBTM_INQ_RESULTS_CB *p_results_cb, tBTM_CMPL_CB *p_cmpl_cb, tBTM_INQ_DIS_CB *p_discard_cb)
 {
     tBTM_BLE_INQ_CB *p_inq = &btm_cb.ble_ctr_cb.inq_var;
     tBTM_STATUS status = BTM_WRONG_MODE;
@@ -457,6 +457,7 @@ tBTM_STATUS BTM_BleScan(BOOLEAN start, UINT32 duration,
 
         btm_cb.ble_ctr_cb.p_scan_results_cb = p_results_cb;
         btm_cb.ble_ctr_cb.p_scan_cmpl_cb = p_cmpl_cb;
+        btm_cb.ble_ctr_cb.p_obs_discard_cb = p_discard_cb;
         status = BTM_CMD_STARTED;
 
         /* scan is not started */
@@ -708,6 +709,37 @@ extern void BTM_BleReadControllerFeatures(tBTM_BLE_CTRL_FEATURES_CBACK  *p_vsc_c
     return ;
 }
 
+void BTM_VendorHciEchoCmdCallback(tBTM_VSC_CMPL *p1)
+{
+    if (!p1) {
+        return;
+    }
+    uint8_t *p = p1->p_param_buf;
+    uint8_t status, echo;
+    STREAM_TO_UINT8  (status, p);
+    STREAM_TO_UINT8  (echo, p);
+    BTM_TRACE_DEBUG("%s status 0x%x echo 0x%x", __func__, status, echo);
+}
+
+/******************************************************************************
+**
+** Function         BTM_VendorHciEchoCmdTest
+**
+** Description      vendor common echo hci cmd test, controller will return status and echo
+**
+** Parameters:      echo : echo value
+**
+** Returns          void
+**
+*******************************************************************************/
+void BTM_VendorHciEchoCmdTest(uint8_t echo)
+{
+    BTM_VendorSpecificCommand (HCI_VENDOR_COMMON_ECHO_CMD_OPCODE,
+                                1,
+                                &echo, 
+                                BTM_VendorHciEchoCmdCallback);
+}
+
 /*******************************************************************************
 **
 ** Function         BTM_BleEnableMixedPrivacyMode
@@ -1615,6 +1647,52 @@ tBTM_STATUS BTM_BleWriteScanRspRaw(UINT8 *p_raw_scan_rsp, UINT32 raw_scan_rsp_le
     }
 }
 
+/*******************************************************************************
+**
+** Function         BTM_UpdateBleDuplicateExceptionalList
+**
+** Description      This function is called to update duplicate scan exceptional list.
+**
+** Parameters:      subcode: add, remove or clean duplicate scan exceptional list.
+**                  type: device info type
+**                  device_info: device information
+**                  update_exceptional_list_cmp_cb: complete callback
+**
+** Returns          status
+**
+*******************************************************************************/
+tBTM_STATUS BTM_UpdateBleDuplicateExceptionalList(uint8_t subcode, uint32_t type, BD_ADDR device_info, 
+                                                tBTM_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_CMPL_CBACK update_exceptional_list_cmp_cb)
+{
+    tBTM_BLE_CB *ble_cb = &btm_cb.ble_ctr_cb;
+    ble_cb->update_exceptional_list_cmp_cb = update_exceptional_list_cmp_cb;
+    tBTM_STATUS status = BTM_NO_RESOURCES;
+    if (!controller_get_interface()->supports_ble()) {
+        return BTM_ILLEGAL_VALUE;
+    }
+    if(!device_info) {
+        return BTM_ILLEGAL_VALUE;
+    }
+    // subcoe + type + device info
+    uint8_t device_info_array[1 + 4 + BD_ADDR_LEN] = {0};
+    device_info_array[0] = subcode;
+    device_info_array[1] = type & 0xff;
+    device_info_array[2] = (type >> 8) & 0xff;
+    device_info_array[3] = (type >> 16) & 0xff;
+    device_info_array[4] = (type >> 24) & 0xff;
+    if(type == BTM_DUPLICATE_SCAN_EXCEPTIONAL_INFO_ADV_ADDR) {
+        bt_rcopy(&device_info_array[5], device_info, BD_ADDR_LEN);
+    } else {
+        memcpy(&device_info_array[5], device_info, 4);
+    }
+    status = BTM_VendorSpecificCommand(HCI_VENDOR_BLE_UPDATE_DUPLICATE_EXCEPTIONAL_LIST, 1 + 4 + BD_ADDR_LEN, device_info_array, NULL);
+    if(status == BTM_CMD_STARTED) {
+        status = BTM_SUCCESS;
+    }
+
+    return status;
+}
+
 /*******************************************************************************
 **
 ** Function         BTM_BleWriteAdvData
@@ -3436,6 +3514,17 @@ static void btm_ble_process_adv_pkt_cont(BD_ADDR bda, UINT8 addr_type, UINT8 evt
     }
 }
 
+void btm_ble_process_adv_discard_evt(UINT8 *p)
+{
+#if (BLE_ADV_REPORT_FLOW_CONTROL == TRUE)
+    uint32_t num_dis = 0;
+    STREAM_TO_UINT32 (num_dis, p);
+    tBTM_INQ_DIS_CB *p_obs_discard_cb = btm_cb.ble_ctr_cb.p_obs_discard_cb;
+    if(p_obs_discard_cb) {
+        (p_obs_discard_cb)(num_dis);
+    }
+#endif
+}
 /*******************************************************************************
 **
 ** Function         btm_ble_start_scan

components/bt/bluedroid/stack/btm/btm_devctl.c

@@ -689,6 +689,21 @@ tBTM_STATUS BTM_VendorSpecificCommand(UINT16 opcode, UINT8 param_len,
 void btm_vsc_complete (UINT8 *p, UINT16 opcode, UINT16 evt_len,
                        tBTM_CMPL_CB *p_vsc_cplt_cback)
 {
+    tBTM_BLE_CB *ble_cb = &btm_cb.ble_ctr_cb;
+    switch(opcode) {
+        case HCI_VENDOR_BLE_UPDATE_DUPLICATE_EXCEPTIONAL_LIST: {
+            uint8_t subcode, status; uint32_t length;
+            STREAM_TO_UINT8(status, p);
+            STREAM_TO_UINT8(subcode, p);
+            STREAM_TO_UINT32(length, p);
+            if(ble_cb && ble_cb->update_exceptional_list_cmp_cb) {
+                (*ble_cb->update_exceptional_list_cmp_cb)(status, subcode, length, p);
+            }
+            break;
+        }
+        default:
+        break;
+    }
     tBTM_VSC_CMPL   vcs_cplt_params;
 
     /* If there was a callback address for vcs complete, call it */

components/bt/bluedroid/stack/btm/include/btm_ble_int.h

@@ -280,6 +280,9 @@ typedef UINT16 tBTM_BLE_STATE_MASK;
 #define BTM_LE_RESOLVING_LIST_MAX     0x20
 #endif
 
+#define BTM_DUPLICATE_SCAN_EXCEPTIONAL_INFO_ADV_ADDR   0
+#define BTM_DUPLICATE_SCAN_EXCEPTIONAL_INFO_MESH_LINK_ID   1
+
 typedef struct {
     BD_ADDR         *resolve_q_random_pseudo;
     UINT8           *resolve_q_action;
@@ -317,6 +320,7 @@ typedef struct {
     /* observer callback and timer */
     tBTM_INQ_RESULTS_CB *p_obs_results_cb;
     tBTM_CMPL_CB *p_obs_cmpl_cb;
+    tBTM_INQ_DIS_CB *p_obs_discard_cb;
     TIMER_LIST_ENT obs_timer_ent;
 
     /* scan callback and timer */
@@ -357,6 +361,7 @@ typedef struct {
     /* current BLE link state */
     tBTM_BLE_STATE_MASK cur_states; /* bit mask of tBTM_BLE_STATE */
     UINT8 link_count[2]; /* total link count master and slave*/
+    tBTM_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_CMPL_CBACK *update_exceptional_list_cmp_cb;
 } tBTM_BLE_CB;
 
 #ifdef __cplusplus
@@ -365,6 +370,7 @@ extern "C" {
 
 void btm_ble_timeout(TIMER_LIST_ENT *p_tle);
 void btm_ble_process_adv_pkt (UINT8 *p);
+void btm_ble_process_adv_discard_evt(UINT8 *p);
 void btm_ble_proc_scan_rsp_rpt (UINT8 *p);
 tBTM_STATUS btm_ble_read_remote_name(BD_ADDR remote_bda, tBTM_INQ_INFO *p_cur, tBTM_CMPL_CB *p_cb);
 BOOLEAN btm_ble_cancel_remote_name(BD_ADDR remote_bda);

components/bt/bluedroid/stack/btu/btu_hcif.c

@@ -122,6 +122,7 @@ static void btu_hcif_ssr_evt (UINT8 *p, UINT16 evt_len);
 #if BLE_INCLUDED == TRUE
 static void btu_ble_ll_conn_complete_evt (UINT8 *p, UINT16 evt_len);
 static void btu_ble_process_adv_pkt (UINT8 *p);
+static void btu_ble_process_adv_dis(UINT8 *p);
 static void btu_ble_read_remote_feat_evt (UINT8 *p);
 static void btu_ble_ll_conn_param_upd_evt (UINT8 *p, UINT16 evt_len);
 static void btu_ble_ll_get_conn_param_format_err_from_contoller (UINT8 status, UINT16 handle);
@@ -331,6 +332,9 @@ void btu_hcif_process_event (UNUSED_ATTR UINT8 controller_id, BT_HDR *p_msg)
         case HCI_BLE_ADV_PKT_RPT_EVT: /* result of inquiry */
             btu_ble_process_adv_pkt(p);
             break;
+        case HCI_BLE_ADV_DISCARD_REPORT_EVT:
+            btu_ble_process_adv_dis(p);
+            break;
         case HCI_BLE_CONN_COMPLETE_EVT:
             btu_ble_ll_conn_complete_evt(p, hci_evt_len);
             break;
@@ -1764,6 +1768,11 @@ static void btu_ble_process_adv_pkt (UINT8 *p)
     btm_ble_process_adv_pkt(p);
 }
 
+static void btu_ble_process_adv_dis(UINT8 *p)
+{
+    btm_ble_process_adv_discard_evt(p);
+}
+
 static void btu_ble_ll_conn_complete_evt ( UINT8 *p, UINT16 evt_len)
 {
     btm_ble_conn_complete(p, evt_len, FALSE);

components/bt/bluedroid/stack/hcic/hciblecmds.c

@@ -999,5 +999,27 @@ BOOLEAN btsnd_hcic_ble_set_data_length(UINT16 conn_handle, UINT16 tx_octets, UIN
     return TRUE;
 }
 
+BOOLEAN btsnd_hcic_ble_update_adv_report_flow_control (UINT16 num)
+{
+    BT_HDR *p;
+    UINT8 *pp;
+
+    if ((p = HCI_GET_CMD_BUF (1)) == NULL) {
+        return (FALSE);
+    }
+
+    pp = (UINT8 *)(p + 1);
+
+    p->len = HCIC_PREAMBLE_SIZE + 2;
+    p->offset = 0;
+
+    UINT16_TO_STREAM (pp, HCI_VENDOR_BLE_ADV_REPORT_FLOW_CONTROL);
+    UINT8_TO_STREAM (pp, 2);
+    UINT16_TO_STREAM (pp, num);
+
+    btu_hcif_send_cmd (LOCAL_BR_EDR_CONTROLLER_ID, p);
+    return TRUE;
+}
+
 #endif
 

components/bt/bluedroid/stack/include/stack/bt_types.h

@@ -790,4 +790,22 @@ static inline void bdsetany(BD_ADDR a)
 {
     bdcpy(a, bd_addr_any);
 }
+
+/*******************************************************************************
+**
+** Function         bt_rcopy
+**
+** Description      memory reverse and copy.
+**
+**
+** Returns          void
+**
+*******************************************************************************/
+static inline void bt_rcopy(UINT8 *dst, UINT8 const *src, UINT16 len)
+{
+    src += len;
+    while (len --) {
+        *dst++ = *--src;
+    }
+}
 #endif

components/bt/bluedroid/stack/include/stack/btm_api.h

@@ -169,6 +169,8 @@ typedef void (tBTM_VS_EVT_CB) (UINT8 len, UINT8 *p);
 */
 typedef void (tBTM_CMPL_CB) (void *p1);
 
+typedef void (tBTM_INQ_DIS_CB) (uint32_t num_dis);
+
 /* VSC callback function for notifying an application that a synchronous
 ** BTM function is complete. The pointer contains the address of any returned data.
 */
@@ -677,7 +679,6 @@ typedef struct {
     UINT8       num_resp;       /* Number of results from the current inquiry */
 } tBTM_INQUIRY_CMPL;
 
-
 /* Structure returned with remote name  request */
 typedef struct {
     UINT16      status;

components/bt/bluedroid/stack/include/stack/btm_ble_api.h

@@ -867,6 +867,7 @@ tBTM_BLE_SCAN_SETUP_CBACK bta_ble_scan_setup_cb;
 typedef void (tBTM_START_ADV_CMPL_CBACK) (UINT8 status);
 typedef void (tBTM_START_STOP_ADV_CMPL_CBACK) (UINT8 status);
 
+typedef void (tBTM_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_CMPL_CBACK) (tBTM_STATUS status, uint8_t subcode, uint32_t length, uint8_t *device_info);
 
 
 /*****************************************************************************
@@ -1242,7 +1243,7 @@ tBTM_STATUS BTM_BleObserve(BOOLEAN start, UINT32 duration,
 *******************************************************************************/
 //extern
 tBTM_STATUS BTM_BleScan(BOOLEAN start, UINT32 duration,
-                           tBTM_INQ_RESULTS_CB *p_results_cb, tBTM_CMPL_CB *p_cmpl_cb);
+                           tBTM_INQ_RESULTS_CB *p_results_cb, tBTM_CMPL_CB *p_cmpl_cb, tBTM_INQ_DIS_CB *p_discard_cb);
 
 
 /*******************************************************************************
@@ -2053,6 +2054,22 @@ tBTM_STATUS BTM_BleGetEnergyInfo(tBTM_BLE_ENERGY_INFO_CBACK *p_ener_cback);
 //extern
 tBTM_STATUS BTM_SetBleDataLength(BD_ADDR bd_addr, UINT16 tx_pdu_length);
 
+/*******************************************************************************
+**
+** Function         BTM_UpdateBleDuplicateExceptionalList
+**
+** Description      This function is called to update duplicate scan exceptional list.
+**
+** Parameters:      subcode: add, remove or clean duplicate scan exceptional list.
+**                  type: device info type
+**                  device_info: device information
+**                  update_exceptional_list_cmp_cb: complete callback
+**
+** Returns          status
+**
+*******************************************************************************/
+
+tBTM_STATUS BTM_UpdateBleDuplicateExceptionalList(uint8_t subcode, uint32_t type, BD_ADDR device_info, tBTM_UPDATE_DUPLICATE_EXCEPTIONAL_LIST_CMPL_CBACK update_exceptional_list_cmp_cb);
 /*
 #ifdef __cplusplus
 }

components/bt/bluedroid/stack/include/stack/hcidefs.h

@@ -45,7 +45,7 @@
 #define HCI_GRP_VENDOR_SPECIFIC         (0x3F << 10)            /* 0xFC00 */
 
 /* Group occupies high 6 bits of the HCI command rest is opcode itself */
-#define HCI_OGF(p)  (UINT8)((0xFC00 & (p)) >> 10)
+#define HCI_OGF(p)  (UINT8)(0x003F & (p >> 10))
 #define HCI_OCF(p)  ( 0x3FF & (p))
 
 /*
@@ -352,27 +352,74 @@
 #define HCI_BLE_READ_RESOLVABLE_ADDR_LOCAL  (0x002C | HCI_GRP_BLE_CMDS)
 #define HCI_BLE_SET_ADDR_RESOLUTION_ENABLE  (0x002D | HCI_GRP_BLE_CMDS)
 #define HCI_BLE_SET_RAND_PRIV_ADDR_TIMOUT   (0x002E | HCI_GRP_BLE_CMDS)
+// Vendor OGF define
+#define HCI_VENDOR_OGF          0x3F
 
-/* LE Get Vendor Capabilities Command OCF */
-#define HCI_BLE_VENDOR_CAP_OCF    (0x0153 | HCI_GRP_VENDOR_SPECIFIC)
+// ESP vendor group define
+#define HCI_ESP_GROUP_COMMON    0x01
+#define HCI_ESP_GROUP_BLE       0x02
+#define HCI_ESP_GROUP_BT        0x03
+#define HCI_ESP_GROUP_END       0x07
 
+//ESP common subcode define
+#define HCI_SUBCODE_COMMON_INIT 0x00
+#define HCI_SUBCODE_COMMON_ECHO 0x01
+#define HCI_SUBCODE_COMMON_MAX  0x7F
+
+//ESP BLE subcode define
+#define HCI_SUBCODE_BLE_INIT                       0x00
+#define HCI_SUBCODE_BLE_MULTI_ADV                  0x01
+#define HCI_SUBCODE_BLE_BATCH_SCAN                 0x02
+#define HCI_SUBCODE_BLE_ADV_FILTER                 0x03
+#define HCI_SUBCODE_BLE_TRACK_ADV                  0x04
+#define HCI_SUBCODE_BLE_ENERGY_INFO                0x05
+#define HCI_SUBCODE_BLE_EXTENDED_SCAN_PARAMS       0x06
+#define HCI_SUBCODE_BLE_LONG_ADV                   0x07
+#define HCI_SUBCODE_BLE_DUPLICATE_EXCEPTIONAL_LIST 0x08
+#define HCI_SUBCODE_BLE_SET_ADV_FLOW_CONTROL       0x09
+#define HCI_SUBCODE_BLE_ADV_REPORT_FLOW_CONTROL    0x0A
+#define HCI_SUBCODE_BLE_MAX                        0x7F
+
+//ESP BT subcode define
+#define HCI_SUBCODE_BT_INIT     0x00
+#define HCI_SUBCODE_BT_MAX      0x7F
+
+#define HCI_ESP_VENDOR_OPCODE_BUILD(ogf, group, subcode) ((ogf << 10) | (group <<7) | (subcode << 0))
+/*
+* |     OGF     | VENDIOR GROUP | GROUP SUBCODE |
+* | 1 1 1 1 1 1 |     0 0 0     | X X X X X X X | Already Exist
+* | 1 1 1 1 1 1 |     0 0 1     | X X X X X X X | ESP VENDOR COMMON HCI CMD
+* | 1 1 1 1 1 1 |     0 1 0     | X X X X X X X | ESP VENDOR BLE HCI CMD
+* | 1 1 1 1 1 1 |     0 1 1     | X X X X X X X | ESP VENDOR BT HCI CMD
+* | 1 1 1 1 1 1 |     1 0 0     | X X X X X X X | RESERVED FOR FUTURE USE
+* | 1 1 1 1 1 1 |     1 0 1     | X X X X X X X | RESERVED FOR FUTURE USE
+* | 1 1 1 1 1 1 |     1 1 0     | X X X X X X X | RESERVED FOR FUTURE USE
+* | 1 1 1 1 1 1 |     1 1 1     | X X X X X X X | RESERVED FOR FUTURE USE
+*/
+
+// ESP COMMON HCI CMD
+#define HCI_VENDOR_COMMON_ECHO_CMD_OPCODE HCI_ESP_VENDOR_OPCODE_BUILD(HCI_VENDOR_OGF, HCI_ESP_GROUP_COMMON, HCI_SUBCODE_COMMON_ECHO)
+
+//ESP BLE HCI CMD
 /* Multi adv OCF */
-#define HCI_BLE_MULTI_ADV_OCF     (0x0154 | HCI_GRP_VENDOR_SPECIFIC)
-
+#define HCI_BLE_MULTI_ADV_OCF             HCI_ESP_VENDOR_OPCODE_BUILD(HCI_VENDOR_OGF, HCI_ESP_GROUP_COMMON, HCI_SUBCODE_BLE_MULTI_ADV)
 /* Batch scan OCF */
-#define HCI_BLE_BATCH_SCAN_OCF    (0x0156 | HCI_GRP_VENDOR_SPECIFIC)
-
+#define HCI_BLE_BATCH_SCAN_OCF            HCI_ESP_VENDOR_OPCODE_BUILD(HCI_VENDOR_OGF, HCI_ESP_GROUP_COMMON, HCI_SUBCODE_BLE_BATCH_SCAN)
 /* ADV filter OCF */
-#define HCI_BLE_ADV_FILTER_OCF    (0x0157 | HCI_GRP_VENDOR_SPECIFIC)
-
+#define HCI_BLE_ADV_FILTER_OCF            HCI_ESP_VENDOR_OPCODE_BUILD(HCI_VENDOR_OGF, HCI_ESP_GROUP_COMMON, HCI_SUBCODE_BLE_ADV_FILTER)
 /* Tracking OCF */
-#define HCI_BLE_TRACK_ADV_OCF     (0x0158 | HCI_GRP_VENDOR_SPECIFIC)
-
+#define HCI_BLE_TRACK_ADV_OCF             HCI_ESP_VENDOR_OPCODE_BUILD(HCI_VENDOR_OGF, HCI_ESP_GROUP_COMMON, HCI_SUBCODE_BLE_TRACK_ADV)
 /* Energy info OCF */
-#define HCI_BLE_ENERGY_INFO_OCF   (0x0159 | HCI_GRP_VENDOR_SPECIFIC)
-
+#define HCI_BLE_ENERGY_INFO_OCF           HCI_ESP_VENDOR_OPCODE_BUILD(HCI_VENDOR_OGF, HCI_ESP_GROUP_COMMON, HCI_SUBCODE_BLE_ENERGY_INFO)
 /* Extended BLE Scan parameters OCF */
-#define HCI_BLE_EXTENDED_SCAN_PARAMS_OCF  (0x0160 | HCI_GRP_VENDOR_SPECIFIC)
+#define HCI_BLE_EXTENDED_SCAN_PARAMS_OCF  HCI_ESP_VENDOR_OPCODE_BUILD(HCI_VENDOR_OGF, HCI_ESP_GROUP_BLE, HCI_SUBCODE_BLE_EXTENDED_SCAN_PARAMS)
+/* Long BLE Adv data OCF */
+#define HCI_VENDOR_BLE_LONG_ADV_DATA      HCI_ESP_VENDOR_OPCODE_BUILD(HCI_VENDOR_OGF, HCI_ESP_GROUP_BLE, HCI_SUBCODE_BLE_LONG_ADV)
+/* BLE update duplicate scan exceptional list */
+#define HCI_VENDOR_BLE_UPDATE_DUPLICATE_EXCEPTIONAL_LIST  HCI_ESP_VENDOR_OPCODE_BUILD(HCI_VENDOR_OGF, HCI_ESP_GROUP_BLE, HCI_SUBCODE_BLE_DUPLICATE_EXCEPTIONAL_LIST)
+#define HCI_VENDOR_BLE_SET_ADV_FLOW_CONTROL        HCI_ESP_VENDOR_OPCODE_BUILD(HCI_VENDOR_OGF, HCI_ESP_GROUP_BLE, HCI_SUBCODE_BLE_SET_ADV_FLOW_CONTROL)
+#define HCI_VENDOR_BLE_ADV_REPORT_FLOW_CONTROL     HCI_ESP_VENDOR_OPCODE_BUILD(HCI_VENDOR_OGF, HCI_ESP_GROUP_BLE, HCI_SUBCODE_BLE_ADV_REPORT_FLOW_CONTROL)
+//ESP BT HCI CMD
 
 /* subcode for multi adv feature */
 #define BTM_BLE_MULTI_ADV_SET_PARAM                     0x01
@@ -706,6 +753,8 @@
 #define HCI_BLE_DATA_LENGTH_CHANGE_EVT      0x07
 #define HCI_BLE_ENHANCED_CONN_COMPLETE_EVT  0x0a
 #define HCI_BLE_DIRECT_ADV_EVT              0x0b
+/* ESP vendor BLE Event sub code */
+#define HCI_BLE_ADV_DISCARD_REPORT_EVT      0XF0
 
 /* Definitions for LE Channel Map */
 #define HCI_BLE_CHNL_MAP_SIZE               5

components/bt/bluedroid/stack/include/stack/hcimsgs.h

@@ -555,6 +555,10 @@ BOOLEAN btsnd_hcic_write_voice_settings(UINT16 flags);            /* Write Voice
 #define HCI_HOST_FLOW_CTRL_SCO_ON       2
 #define HCI_HOST_FLOW_CTRL_BOTH_ON      3
 
+#define  HCI_HOST_FLOW_CTRL_ADV_REPORT_OFF     0
+#define  HCI_HOST_FLOW_CTRL_ADV_REPORT_ON      1
+
+
 BOOLEAN btsnd_hcic_write_auto_flush_tout(UINT16 handle,
         UINT16 timeout);    /* Write Retransmit Timout */
 
@@ -806,6 +810,8 @@ BOOLEAN btsnd_hcic_read_authenticated_payload_tout(UINT16 handle);
 BOOLEAN btsnd_hcic_write_authenticated_payload_tout(UINT16 handle,
         UINT16 timeout);
 
+BOOLEAN btsnd_hcic_ble_update_adv_report_flow_control (UINT16 num);
+
 #define HCIC_PARAM_SIZE_WRITE_AUTHENT_PAYLOAD_TOUT  4
 
 #define HCI__WRITE_AUTHENT_PAYLOAD_TOUT_HANDLE_OFF  0

components/console/linenoise/linenoise.c

@@ -106,6 +106,7 @@
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
+#include <stdio_ext.h>
 #include <errno.h>
 #include <string.h>
 #include <stdlib.h>
@@ -879,6 +880,9 @@ static int linenoiseEdit(char *buf, size_t buflen, const char *prompt)
             linenoiseEditDeletePrevWord(&l);
             break;
         }
+        if (__fbufsize(stdout) > 0) {
+            fflush(stdout);
+        }
     }
     return l.len;
 }

components/driver/gpio.c

@@ -539,6 +539,20 @@ esp_err_t gpio_hold_dis(gpio_num_t gpio_num)
     return r == ESP_OK ? ESP_OK : ESP_ERR_NOT_SUPPORTED;
 }
 
+void gpio_deep_sleep_hold_en(void)
+{
+    portENTER_CRITICAL(&gpio_spinlock);
+    SET_PERI_REG_MASK(RTC_CNTL_DIG_ISO_REG, RTC_CNTL_DG_PAD_AUTOHOLD_EN_M);
+    portEXIT_CRITICAL(&gpio_spinlock);
+}
+
+void gpio_deep_sleep_hold_dis(void)
+{
+    portENTER_CRITICAL(&gpio_spinlock);
+    CLEAR_PERI_REG_MASK(RTC_CNTL_DIG_ISO_REG, RTC_CNTL_DG_PAD_AUTOHOLD_EN_M);
+    portEXIT_CRITICAL(&gpio_spinlock);
+}
+
 void gpio_iomux_in(uint32_t gpio, uint32_t signal_idx)
 {
     GPIO.func_in_sel_cfg[signal_idx].sig_in_sel = 0;

components/driver/include/driver/gpio.h

@@ -537,13 +537,17 @@ esp_err_t gpio_set_drive_capability(gpio_num_t gpio_num, gpio_drive_cap_t streng
 esp_err_t gpio_get_drive_capability(gpio_num_t gpio_num, gpio_drive_cap_t* strength);
 
 /**
-  * @brief Set gpio pad hold function.
+  * @brief Enable gpio pad hold function.
   *
   * The gpio pad hold function works in both input and output modes, but must be output-capable gpios.
   * If pad hold enabled:
   *   in output mode: the output level of the pad will be force locked and can not be changed.
   *   in input mode: the input value read will not change, regardless the changes of input signal.
   *
+  * The state of digital gpio cannot be held during Deep-sleep, and it will resume the hold function
+  * when the chip wakes up from Deep-sleep. If the digital gpio also needs to be held during Deep-sleep,
+  * `gpio_deep_sleep_hold_en` should also be called.
+  *
   * Power down or call gpio_hold_dis will disable this function.
   *
   * @param gpio_num GPIO number, only support output-capable GPIOs
@@ -555,7 +559,15 @@ esp_err_t gpio_get_drive_capability(gpio_num_t gpio_num, gpio_drive_cap_t* stren
 esp_err_t gpio_hold_en(gpio_num_t gpio_num);
 
 /**
-  * @brief Unset gpio pad hold function.
+  * @brief Disable gpio pad hold function.
+  *
+  * When the chip is woken up from Deep-sleep, the gpio will be set to the default mode, so, the gpio will output
+  * the default level if this function is called. If you dont't want the level changes, the gpio should be configured to
+  * a known state before this function is called.
+  *  e.g.
+  *     If you hold gpio18 high during Deep-sleep, after the chip is woken up and `gpio_hold_dis` is called, 
+  *     gpio18 will output low level(because gpio18 is input mode by default). If you don't want this behavior,
+  *     you should configure gpio18 as output mode and set it to hight level before calling `gpio_hold_dis`.
   *
   * @param gpio_num GPIO number, only support output-capable GPIOs
   *
@@ -563,7 +575,24 @@ esp_err_t gpio_hold_en(gpio_num_t gpio_num);
   *     - ESP_OK Success
   *     - ESP_ERR_NOT_SUPPORTED Not support pad hold function
   */
- esp_err_t gpio_hold_dis(gpio_num_t gpio_num);
+esp_err_t gpio_hold_dis(gpio_num_t gpio_num);
+
+/**
+  * @brief Enable all digital gpio pad hold function during Deep-sleep.
+  *
+  * When the chip is in Deep-sleep mode, all digital gpio will hold the state before sleep, and when the chip is woken up,
+  * the status of digital gpio will not be held. Note that the pad hold feature only works when the chip is in Deep-sleep mode,
+  * when not in sleep mode, the digital gpio state can be changed even you have called this function.
+  *
+  * Power down or call gpio_hold_dis will disable this function, otherwise, the digital gpio hold feature works as long as the chip enter Deep-sleep.
+  */
+void gpio_deep_sleep_hold_en(void);
+
+/**
+  * @brief Disable all digital gpio pad hold function during Deep-sleep.
+  *
+  */
+void gpio_deep_sleep_hold_dis(void);
 
 /**
   * @brief Set pad input to a peripheral signal through the IOMUX.

components/driver/include/driver/rmt.h

@@ -135,16 +135,11 @@ typedef struct {
  *        This function may be called from an ISR, so, the code should be short and efficient.
  *
  * @param  src Pointer to the buffer storing the raw data that needs to be converted to rmt format.
- *
  * @param[out] dest Pointer to the buffer storing the rmt format data.
- *
  * @param  src_size The raw data size.
- *
  * @param  wanted_num The number of rmt format data that wanted to get.
- *
  * @param[out] translated_size The size of the raw data that has been converted to rmt format,
  *             it should return 0 if no data is converted in user callback.
- *
  * @param[out] item_num The number of the rmt format data that actually converted to, it can be less than wanted_num if there is not enough raw data,
  *             but cannot exceed wanted_num. it should return 0 if no data was converted.
  *
@@ -159,7 +154,6 @@ typedef void (*sample_to_rmt_t)(const void* src, rmt_item32_t* dest, size_t src_
  * @brief Set RMT clock divider, channel clock is divided from source clock.
  *
  * @param channel RMT channel (0-7)
- *
  * @param div_cnt RMT counter clock divider
  *
  * @return
@@ -172,7 +166,6 @@ esp_err_t rmt_set_clk_div(rmt_channel_t channel, uint8_t div_cnt);
  * @brief Get RMT clock divider, channel clock is divided from source clock.
  *
  * @param channel RMT channel (0-7)
- *
  * @param div_cnt pointer to accept RMT counter divider
  *
  * @return
@@ -189,7 +182,6 @@ esp_err_t rmt_get_clk_div(rmt_channel_t channel, uint8_t* div_cnt);
  *        the receive process is finished.
  *
  * @param channel RMT channel (0-7)
- *
  * @param thresh RMT RX idle threshold
  *
  * @return
@@ -206,7 +198,6 @@ esp_err_t rmt_set_rx_idle_thresh(rmt_channel_t channel, uint16_t thresh);
  *        the receive process is finished.
  *
  * @param channel RMT channel (0-7)
- *
  * @param thresh pointer to accept RMT RX idle threshold value
  *
  * @return
@@ -234,7 +225,6 @@ esp_err_t rmt_get_rx_idle_thresh(rmt_channel_t channel, uint16_t *thresh);
  *        Channel 0 can use at most 8 blocks of memory, accordingly channel 7 can only use one memory block.
  *
  * @param channel RMT channel (0-7)
- *
  * @param rmt_mem_num RMT RX memory block number, one block has 64 * 32 bits.
  *
  * @return
@@ -247,7 +237,6 @@ esp_err_t rmt_set_mem_block_num(rmt_channel_t channel, uint8_t rmt_mem_num);
  * @brief Get RMT memory block number
  *
  * @param channel RMT channel (0-7)
- *
  * @param rmt_mem_num Pointer to accept RMT RX memory block number
  *
  * @return
@@ -263,13 +252,9 @@ esp_err_t rmt_get_mem_block_num(rmt_channel_t channel, uint8_t* rmt_mem_num);
  *        The unit of carrier_high/low is the source clock tick, not the divided channel counter clock.
  *
  * @param channel RMT channel (0-7)
- *
  * @param carrier_en Whether to enable output carrier.
- *
  * @param high_level High level duration of carrier
- *
  * @param low_level Low level duration of carrier.
- *
  * @param carrier_level Configure the way carrier wave is modulated for channel 0-7.
  *     - 1'b1:transmit on low output level
  *     - 1'b0:transmit on high output level
@@ -286,7 +271,6 @@ esp_err_t rmt_set_tx_carrier(rmt_channel_t channel, bool carrier_en, uint16_t hi
  *        Reduce power consumed by memory. 1:memory is in low power state.
  *
  * @param channel RMT channel (0-7)
- *
  * @param pd_en RMT memory low power enable.
  *
  * @return
@@ -299,7 +283,6 @@ esp_err_t rmt_set_mem_pd(rmt_channel_t channel, bool pd_en);
  * @brief Get RMT memory low power mode.
  *
  * @param channel RMT channel (0-7)
- *
  * @param pd_en Pointer to accept RMT memory low power mode.
  *
  * @return
@@ -312,7 +295,6 @@ esp_err_t rmt_get_mem_pd(rmt_channel_t channel, bool* pd_en);
  * @brief Set RMT start sending data from memory.
  *
  * @param channel RMT channel (0-7)
- *
  * @param tx_idx_rst Set true to reset memory index for TX.
  *                   Otherwise, transmitter will continue sending from the last index in memory.
  *
@@ -337,7 +319,6 @@ esp_err_t rmt_tx_stop(rmt_channel_t channel);
  * @brief Set RMT start receiving data.
  *
  * @param channel RMT channel (0-7)
- *
  * @param rx_idx_rst Set true to reset memory index for receiver.
  *                   Otherwise, receiver will continue receiving data to the last index in memory.
  *
@@ -373,7 +354,6 @@ esp_err_t rmt_memory_rw_rst(rmt_channel_t channel);
  * @brief Set RMT memory owner.
  *
  * @param channel RMT channel (0-7)
- *
  * @param owner To set when the transmitter or receiver can process the memory of channel.
  *
  * @return
@@ -386,7 +366,6 @@ esp_err_t rmt_set_memory_owner(rmt_channel_t channel, rmt_mem_owner_t owner);
  * @brief Get RMT memory owner.
  *
  * @param channel RMT channel (0-7)
- *
  * @param owner Pointer to get memory owner.
  *
  * @return
@@ -399,7 +378,6 @@ esp_err_t rmt_get_memory_owner(rmt_channel_t channel, rmt_mem_owner_t* owner);
  * @brief Set RMT tx loop mode.
  *
  * @param channel RMT channel (0-7)
- *
  * @param loop_en Enable RMT transmitter loop sending mode.
  *                If set true, transmitter will continue sending from the first data
  *                to the last data in channel 0-7 over and over again in a loop.
@@ -414,7 +392,6 @@ esp_err_t rmt_set_tx_loop_mode(rmt_channel_t channel, bool loop_en);
  * @brief Get RMT tx loop mode.
  *
  * @param channel RMT channel (0-7)
- *
  * @param loop_en Pointer to accept RMT transmitter loop sending mode.
  *
  * @return
@@ -430,9 +407,7 @@ esp_err_t rmt_get_tx_loop_mode(rmt_channel_t channel, bool* loop_en);
  *        Counted in source clock, not divided counter clock.
  *
  * @param channel RMT channel (0-7)
- *
  * @param rx_filter_en To enable RMT receiver filter.
- *
  * @param thresh Threshold of pulse width for receiver.
  *
  * @return
@@ -449,7 +424,6 @@ esp_err_t rmt_set_rx_filter(rmt_channel_t channel, bool rx_filter_en, uint8_t th
  *        2. REF tick clock, which would be 1Mhz (not supported in this version).
  *
  * @param channel RMT channel (0-7)
- *
  * @param base_clk To choose source clock for RMT module.
  *
  * @return
@@ -466,7 +440,6 @@ esp_err_t rmt_set_source_clk(rmt_channel_t channel, rmt_source_clk_t base_clk);
  *        2. REF tick clock, which would be 1Mhz (not supported in this version).
  *
  * @param channel RMT channel (0-7)
- *
  * @param src_clk Pointer to accept source clock for RMT module.
  *
  * @return
@@ -479,9 +452,7 @@ esp_err_t rmt_get_source_clk(rmt_channel_t channel, rmt_source_clk_t* src_clk);
  * @brief Set RMT idle output level for transmitter
  *
  * @param channel RMT channel (0-7)
- *
  * @param idle_out_en To enable idle level output.
- *
  * @param level To set the output signal's level for channel 0-7 in idle state.
  *
  * @return
@@ -490,11 +461,23 @@ esp_err_t rmt_get_source_clk(rmt_channel_t channel, rmt_source_clk_t* src_clk);
  */
 esp_err_t rmt_set_idle_level(rmt_channel_t channel, bool idle_out_en, rmt_idle_level_t level);
 
+/**
+ * @brief Get RMT idle output level for transmitter
+ *
+ * @param channel RMT channel (0-7)
+ * @param idle_out_en Pointer to accept value of enable idle.
+ * @param level Pointer to accept value of output signal's level in idle state for specified channel.
+ *
+ * @return
+ *     - ESP_ERR_INVALID_ARG Parameter error
+ *     - ESP_OK Success
+ */
+esp_err_t rmt_get_idle_level(rmt_channel_t channel, bool* idle_out_en, rmt_idle_level_t* level);
+
 /**
  * @brief Get RMT status
  *
  * @param channel RMT channel (0-7)
- *
  * @param status Pointer to accept channel status.
  *
  * @return
@@ -523,7 +506,6 @@ void rmt_clr_intr_enable_mask(uint32_t mask);
  * @brief Set RMT RX interrupt enable
  *
  * @param channel RMT channel (0 - 7)
- *
  * @param en enable or disable RX interrupt.
  *
  * @return
@@ -536,7 +518,6 @@ esp_err_t rmt_set_rx_intr_en(rmt_channel_t channel, bool en);
  * @brief Set RMT RX error interrupt enable
  *
  * @param channel RMT channel (0 - 7)
- *
  * @param en enable or disable RX err interrupt.
  *
  * @return
@@ -549,7 +530,6 @@ esp_err_t rmt_set_err_intr_en(rmt_channel_t channel, bool en);
  * @brief Set RMT TX interrupt enable
  *
  * @param channel RMT channel (0 - 7)
- *
  * @param en enable or disable TX interrupt.
  *
  * @return
@@ -564,9 +544,7 @@ esp_err_t rmt_set_tx_intr_en(rmt_channel_t channel, bool en);
  * An interrupt will be triggered when the number of transmitted items reaches the threshold value
  *
  * @param channel RMT channel (0 - 7)
- *
  * @param en enable or disable TX event interrupt.
- *
  * @param evt_thresh RMT event interrupt threshold value
  *
  * @return
@@ -579,9 +557,7 @@ esp_err_t rmt_set_tx_thr_intr_en(rmt_channel_t channel, bool en, uint16_t evt_th
  * @brief Set RMT pin
  *
  * @param channel RMT channel (0 - 7)
- *
  * @param mode TX or RX mode for RMT
- *
  * @param gpio_num GPIO number to transmit or receive the signal.
  *
  * @return
@@ -637,11 +613,8 @@ esp_err_t rmt_isr_deregister(rmt_isr_handle_t handle);
  * @brief Fill memory data of channel with given RMT items.
  *
  * @param channel RMT channel (0 - 7)
- *
  * @param item Pointer of items.
- *
  * @param item_num RMT sending items number.
- *
  * @param mem_offset Index offset of memory.
  *
  * @return
@@ -654,9 +627,7 @@ esp_err_t rmt_fill_tx_items(rmt_channel_t channel, const rmt_item32_t* item, uin
  * @brief Initialize RMT driver
  *
  * @param channel RMT channel (0 - 7)
- *
  * @param rx_buf_size Size of RMT RX ringbuffer. Can be 0 if the RX ringbuffer is not used.
- *
  * @param intr_alloc_flags Flags for the RMT driver interrupt handler. Pass 0 for default flags. See esp_intr_alloc.h for details.
  *        If ESP_INTR_FLAG_IRAM is used, please do not use the memory allocated from psram when calling rmt_write_items.
  *
@@ -685,12 +656,9 @@ esp_err_t rmt_driver_uninstall(rmt_channel_t channel);
  *        This API allows user to send waveform with any length.
  *
  * @param channel RMT channel (0 - 7)
- *
  * @param rmt_item head point of RMT items array.
  *        If ESP_INTR_FLAG_IRAM is used, please do not use the memory allocated from psram when calling rmt_write_items.
- *
  * @param item_num RMT data item number.
- *
  * @param wait_tx_done
  *        - If set 1, it will block the task and wait for sending done.
  *        - If set 0, it will not wait and return immediately.
@@ -714,8 +682,7 @@ esp_err_t rmt_write_items(rmt_channel_t channel, const rmt_item32_t* rmt_item, i
  * @brief Wait RMT TX finished.
  *
  * @param channel RMT channel (0 - 7)
- *
- * @param wait_time Maximum time in ticks to wait for transmission to be complete 
+ * @param wait_time Maximum time in ticks to wait for transmission to be complete.  If set 0, return immediately with ESP_ERR_TIMEOUT if TX is busy (polling).
  *
  * @return
  *     - ESP_OK RMT Tx done successfully
@@ -731,7 +698,6 @@ esp_err_t rmt_wait_tx_done(rmt_channel_t channel, TickType_t wait_time);
  *        Users can get the RMT RX ringbuffer handle, and process the RX data.
  *
  * @param channel RMT channel (0 - 7)
- *
  * @param buf_handle Pointer to buffer handle to accept RX ringbuffer handle.
  *
  * @return
@@ -746,7 +712,6 @@ esp_err_t rmt_get_ringbuf_handle(rmt_channel_t channel, RingbufHandle_t* buf_han
  *        If a channel is initialized more than once, tha user callback will be replaced by the later.
  *
  * @param channel RMT channel (0 - 7).
- *
  * @param fn Point to the data conversion function.
  *
  * @return
@@ -760,11 +725,8 @@ esp_err_t rmt_translator_init(rmt_channel_t channel, sample_to_rmt_t fn);
  *        Requires rmt_translator_init to init the translator first.
  *
  * @param channel RMT channel (0 - 7).
- *
  * @param src Pointer to the raw data.
- *
  * @param src_size The size of the raw data.
- *
  * @param wait_tx_done Set true to wait all data send done.
  *
  * @return

components/driver/rmt.c

@@ -306,6 +306,14 @@ esp_err_t rmt_set_idle_level(rmt_channel_t channel, bool idle_out_en, rmt_idle_l
     return ESP_OK;
 }
 
+esp_err_t rmt_get_idle_level(rmt_channel_t channel, bool* idle_out_en, rmt_idle_level_t* level)
+{
+    RMT_CHECK(channel < RMT_CHANNEL_MAX, RMT_CHANNEL_ERROR_STR, ESP_ERR_INVALID_ARG);
+    *idle_out_en = (bool) (RMT.conf_ch[channel].conf1.idle_out_en);
+    *level = (rmt_idle_level_t) (RMT.conf_ch[channel].conf1.idle_out_lv);
+    return ESP_OK;
+}
+
 esp_err_t rmt_get_status(rmt_channel_t channel, uint32_t* status)
 {
     RMT_CHECK(channel < RMT_CHANNEL_MAX, RMT_CHANNEL_ERROR_STR, ESP_ERR_INVALID_ARG);
@@ -837,7 +845,9 @@ esp_err_t rmt_wait_tx_done(rmt_channel_t channel, TickType_t wait_time)
         return ESP_OK;
     }
     else {
-        ESP_LOGE(RMT_TAG, "Timeout on wait_tx_done");
+        if (wait_time != 0) {  // Don't emit error message if just polling.
+            ESP_LOGE(RMT_TAG, "Timeout on wait_tx_done");
+        }
         return ESP_ERR_TIMEOUT;
     }
 }

components/esp32/CMakeLists.txt

@@ -91,7 +91,10 @@ else()
         )
 
     if(CONFIG_SPIRAM_CACHE_WORKAROUND)
-        add_compile_options(-mfix-esp32-psram-cache-issue)
+        # Note: Adding as a PUBLIC compile option here causes this option to propagate to all components that depend on esp32.
+        #
+        # To handle some corner cases, the same flag is set in project_include.cmake
+        target_compile_options(esp32 PUBLIC -mfix-esp32-psram-cache-issue)
     else()
         target_linker_script(esp32 "ld/esp32.rom.spiram_incompatible_fns.ld")
     endif()

components/esp32/Kconfig

@@ -124,7 +124,8 @@ config SPIRAM_CACHE_WORKAROUND
     help
         Revision 1 of the ESP32 has a bug that can cause a write to PSRAM not to take place in some situations
         when the cache line needs to be fetched from external RAM and an interrupt occurs. This enables a
-        fix in the compiler that makes sure the specific code that is vulnerable to this will not be emitted.
+        fix in the compiler (-mfix-esp32-psram-cache-issue) that makes sure the specific code that is vulnerable
+        to this will not be emitted.
         
         This will also not use any bits of newlib that are located in ROM, opting for a version that is compiled
         with the workaround and located in flash instead.

components/esp32/esp_timer_esp32.c

@@ -97,7 +97,7 @@ static uint32_t s_alarm_overflow_val = DEFAULT_ALARM_OVERFLOW_VAL;
 
 static const char* TAG = "esp_timer_impl";
 
-// Interrupt handle retuned by the interrupt allocator
+// Interrupt handle returned by the interrupt allocator
 static intr_handle_t s_timer_interrupt_handle;
 
 // Function from the upper layer to be called when the interrupt happens.
@@ -123,7 +123,7 @@ static uint32_t s_timer_us_per_overflow;
 // value than the one which caused an interrupt. This can cause interrupt handler
 // to consider that the interrupt has happened due to timer overflow, incrementing
 // s_time_base_us. To avoid this, frequency switch hook sets this flag if
-// it needs to set timer alarm value to ALARM_OVERFLOW_VAL. Interrupt hanler
+// it needs to set timer alarm value to ALARM_OVERFLOW_VAL. Interrupt handler
 // will not increment s_time_base_us if this flag is set.
 static bool s_mask_overflow;
 

components/esp32/ld/esp32.rom.ld

@@ -57,6 +57,7 @@ PROVIDE ( _bss_start_btdm = 0x3ffb8000);
 PROVIDE ( _bss_end_btdm = 0x3ffbff70);
 PROVIDE ( _daylight = 0x3ffae0a4 );
 PROVIDE ( dbg_default_handler = 0x3ff97218 );
+PROVIDE ( dbg_default_state = 0x3ff97220 );
 PROVIDE ( dbg_state = 0x3ffb8d5d );
 PROVIDE ( DebugE256PublicKey_x = 0x3ff97428 );
 PROVIDE ( DebugE256PublicKey_y = 0x3ff97408 );

components/esp32/project_include.cmake

@@ -0,0 +1,8 @@
+if(CONFIG_SPIRAM_CACHE_WORKAROUND)
+    # We do this here as well as in CMakeLists.txt, because targets that
+    # are not part of the ESP-IDF build system (for cases where a generic
+    # non-IDF CMakeLists.txt file is imported into a component) don't depend
+    # on the esp32 component so don't get the extra flag. This handles that case.
+    add_compile_options(-mfix-esp32-psram-cache-issue)
+endif()
+

components/esp32/test/test_esp_timer.c

@@ -24,7 +24,12 @@ static uint32_t s_old_overflow_val;
 static void setup_overflow()
 {
     s_old_overflow_val = esp_timer_impl_get_overflow_val();
-    esp_timer_impl_set_overflow_val(0x7fffff); /* overflow every ~0.1 sec */}
+    /* Overflow every 0.1 sec.
+     * Chosen so that it is 0 modulo s_timer_ticks_per_us (which is 80),
+     * to prevent roundoff error on each overflow.
+     */
+    esp_timer_impl_set_overflow_val(8000000);
+}
 
 static void teardown_overflow()
 {
@@ -404,6 +409,13 @@ TEST_CASE("esp_timer_get_time call takes less than 1us", "[esp_timer]")
     TEST_PERFORMANCE_LESS_THAN(ESP_TIMER_GET_TIME_PER_CALL, "%dns", ns_per_call);
 }
 
+static int64_t IRAM_ATTR __attribute__((noinline)) get_clock_diff()
+{
+    uint64_t hs_time = esp_timer_get_time();
+    uint64_t ref_time = ref_clock_get();
+    return hs_time - ref_time;
+}
+
 TEST_CASE("esp_timer_get_time returns monotonic values", "[esp_timer]")
 {
     typedef struct {
@@ -411,24 +423,33 @@ TEST_CASE("esp_timer_get_time returns monotonic values", "[esp_timer]")
         bool pass;
         int test_cnt;
         int error_cnt;
-        int64_t total_sq_error;
         int64_t max_error;
+        int64_t avg_diff;
+        int64_t dummy;
     } test_state_t;
 
     void timer_test_task(void* arg) {
         test_state_t* state = (test_state_t*) arg;
         state->pass = true;
-        int64_t start_time = ref_clock_get();
-        int64_t delta = esp_timer_get_time() - start_time;
 
-        int64_t now = start_time;
+        /* make sure both functions are in cache */
+        state->dummy = get_clock_diff();
+
+        /* calculate the difference between the two clocks */
+        portDISABLE_INTERRUPTS();
+        int64_t delta = get_clock_diff();
+        portENABLE_INTERRUPTS();
+        int64_t start_time = ref_clock_get();
         int error_repeat_cnt = 0;
-        while (now - start_time < 10000000) {  /* 10 seconds */
-            int64_t hs_now = esp_timer_get_time();
-            now = ref_clock_get();
-            int64_t diff = hs_now - (now + delta);
+        while (ref_clock_get() - start_time < 10000000) {  /* 10 seconds */
+            /* Get values of both clocks again, and check that they are close to 'delta'.
+             * We don't disable interrupts here, because esp_timer_get_time doesn't lock
+             * interrupts internally, so we check if it can get "broken" by a well placed
+             * interrupt.
+             */
+            int64_t diff = get_clock_diff() - delta;
             /* Allow some difference due to rtos tick interrupting task between
-             * getting 'now' and 'ref_now'.
+             * getting 'hs_now' and 'now'.
              */
             if (abs(diff) > 100) {
                 error_repeat_cnt++;
@@ -440,10 +461,11 @@ TEST_CASE("esp_timer_get_time returns monotonic values", "[esp_timer]")
                 printf("diff=%lld\n", diff);
                 state->pass = false;
             }
+            state->avg_diff += diff;
             state->max_error = MAX(state->max_error, abs(diff));
             state->test_cnt++;
-            state->total_sq_error += diff * diff;
         }
+        state->avg_diff /= state->test_cnt;
         xSemaphoreGive(state->done);
         vTaskDelete(NULL);
     }
@@ -460,10 +482,10 @@ TEST_CASE("esp_timer_get_time returns monotonic values", "[esp_timer]")
 
     for (int i = 0; i < portNUM_PROCESSORS; ++i) {
         TEST_ASSERT_TRUE( xSemaphoreTake(done, portMAX_DELAY) );
-        printf("CPU%d: %s test_cnt=%d error_cnt=%d std_error=%d |max_error|=%d\n",
+        printf("CPU%d: %s test_cnt=%d error_cnt=%d avg_diff=%d |max_error|=%d\n",
                 i, states[i].pass ? "PASS" : "FAIL",
                 states[i].test_cnt, states[i].error_cnt,
-                (int) sqrt(states[i].total_sq_error / states[i].test_cnt), (int) states[i].max_error);
+                (int) states[i].avg_diff, (int) states[i].max_error);
     }
 
     vSemaphoreDelete(done);

components/esp_http_server/include/esp_http_server.h

@@ -324,6 +324,18 @@ typedef struct httpd_req {
      * function for freeing the session context, please specify that here.
      */
     httpd_free_ctx_fn_t free_ctx;
+
+    /**
+     * Flag indicating if Session Context changes should be ignored
+     *
+     * By default, if you change the sess_ctx in some URI handler, the http server
+     * will internally free the earlier context (if non NULL), after the URI handler
+     * returns. If you want to manage the allocation/reallocation/freeing of
+     * sess_ctx yourself, set this flag to true, so that the server will not
+     * perform any checks on it. The context will be cleared by the server
+     * (by calling free_ctx or free()) only if the socket gets closed.
+     */
+    bool ignore_sess_ctx_changes;
 } httpd_req_t;
 
 /**
@@ -585,7 +597,7 @@ esp_err_t httpd_sess_set_pending_override(httpd_handle_t hd, int sockfd, httpd_p
  * session socket fd, from within a URI handler, ie. :
  *      httpd_sess_get_ctx(),
  *      httpd_sess_trigger_close(),
- *      httpd_sess_update_timestamp().
+ *      httpd_sess_update_lru_counter().
  *
  * @note    This API is supposed to be called only from the context of
  *          a URI handler where httpd_req_t* request pointer is valid.
@@ -1111,15 +1123,15 @@ void *httpd_get_global_transport_ctx(httpd_handle_t handle);
 esp_err_t httpd_sess_trigger_close(httpd_handle_t handle, int sockfd);
 
 /**
- * @brief   Update timestamp for a given socket
+ * @brief   Update LRU counter for a given socket
  *
- * Timestamps are internally associated with each session to monitor
+ * LRU Counters are internally associated with each session to monitor
  * how recently a session exchanged traffic. When LRU purge is enabled,
  * if a client is requesting for connection but maximum number of
  * sockets/sessions is reached, then the session having the earliest
- * timestamp is closed automatically.
+ * LRU counter is closed automatically.
  *
- * Updating the timestamp manually prevents the socket from being purged
+ * Updating the LRU counter manually prevents the socket from being purged
  * due to the Least Recently Used (LRU) logic, even though it might not
  * have received traffic for some time. This is useful when all open
  * sockets/session are frequently exchanging traffic but the user specifically
@@ -1130,15 +1142,15 @@ esp_err_t httpd_sess_trigger_close(httpd_handle_t handle, int sockfd);
  *          is enabled.
  *
  * @param[in] handle    Handle to server returned by httpd_start
- * @param[in] sockfd    The socket descriptor of the session for which timestamp
+ * @param[in] sockfd    The socket descriptor of the session for which LRU counter
  *                      is to be updated
  *
  * @return
- *  - ESP_OK : Socket found and timestamp updated
+ *  - ESP_OK : Socket found and LRU counter updated
  *  - ESP_ERR_NOT_FOUND   : Socket not found
  *  - ESP_ERR_INVALID_ARG : Null arguments
  */
-esp_err_t httpd_sess_update_timestamp(httpd_handle_t handle, int sockfd);
+esp_err_t httpd_sess_update_lru_counter(httpd_handle_t handle, int sockfd);
 
 /** End of Session
  * @}

components/esp_http_server/src/esp_httpd_priv.h

@@ -118,6 +118,7 @@ typedef enum {
 struct sock_db {
     int fd;                                 /*!< The file descriptor for this socket */
     void *ctx;                              /*!< A custom context for this socket */
+    bool ignore_sess_ctx_changes;           /*!< Flag indicating if session context changes should be ignored */
     void *transport_ctx;                    /*!< A custom 'transport' context for this socket, to be used by send/recv/pending */
     httpd_handle_t handle;                  /*!< Server handle */
     httpd_free_ctx_fn_t free_ctx;      /*!< Function for freeing the context */
@@ -125,7 +126,7 @@ struct sock_db {
     httpd_send_func_t send_fn;              /*!< Send function for this socket */
     httpd_recv_func_t recv_fn;              /*!< Receive function for this socket */
     httpd_pending_func_t pending_fn;        /*!< Pending function for this socket */
-    int64_t timestamp;                      /*!< Timestamp indicating when the socket was last used */
+    uint64_t lru_counter;                   /*!< LRU Counter indicating when the socket was last used */
     char pending_data[PARSER_BLOCK_SIZE];   /*!< Buffer for pending data to be received */
     size_t pending_len;                     /*!< Length of pending data to be received */
 };

components/esp_http_server/src/httpd_parse.c

@@ -537,6 +537,7 @@ static void init_req(httpd_req_t *r, httpd_config_t *config)
     r->user_ctx = 0;
     r->sess_ctx = 0;
     r->free_ctx = 0;
+    r->ignore_sess_ctx_changes = 0;
 }
 
 static void init_req_aux(struct httpd_req_aux *ra, httpd_config_t *config)
@@ -556,13 +557,14 @@ static void httpd_req_cleanup(httpd_req_t *r)
 {
     struct httpd_req_aux *ra = r->aux;
 
-    /* Retrieve session info from the request into the socket database */
-    if (ra->sd->ctx != r->sess_ctx) {
-        /* Free previous context */
+    /* Check if the context has changed and needs to be cleared */ 
+    if ((r->ignore_sess_ctx_changes == false) && (ra->sd->ctx != r->sess_ctx)) {
         httpd_sess_free_ctx(ra->sd->ctx, ra->sd->free_ctx);
-        ra->sd->ctx = r->sess_ctx;
     }
+    /* Retrieve session info from the request into the socket database. */
+    ra->sd->ctx = r->sess_ctx;
     ra->sd->free_ctx = r->free_ctx;
+    ra->sd->ignore_sess_ctx_changes = r->ignore_sess_ctx_changes;
 
     /* Clear out the request and request_aux structures */
     ra->sd = NULL;
@@ -590,6 +592,7 @@ esp_err_t httpd_req_new(struct httpd_data *hd, struct sock_db *sd)
     /* Copy session info to the request */
     r->sess_ctx = sd->ctx;
     r->free_ctx = sd->free_ctx;
+    r->ignore_sess_ctx_changes = sd->ignore_sess_ctx_changes;
     /* Parse request */
     esp_err_t err = httpd_parse_req(hd);
     if (err != ESP_OK) {

components/esp_http_server/src/httpd_sess.c

@@ -195,6 +195,12 @@ static int fd_is_valid(int fd)
     return fcntl(fd, F_GETFD) != -1 || errno != EBADF;
 }
 
+static inline uint64_t httpd_sess_get_lru_counter()
+{
+    static uint64_t lru_counter = 0;
+    return lru_counter++;
+}
+
 void httpd_sess_delete_invalid(struct httpd_data *hd)
 {
     for (int i = 0; i < hd->config.max_open_sockets; i++) {
@@ -297,11 +303,11 @@ esp_err_t httpd_sess_process(struct httpd_data *hd, int newfd)
         return ESP_FAIL;
     }
     ESP_LOGD(TAG, LOG_FMT("success"));
-    sd->timestamp = httpd_os_get_timestamp();
+    sd->lru_counter = httpd_sess_get_lru_counter();
     return ESP_OK;
 }
 
-esp_err_t httpd_sess_update_timestamp(httpd_handle_t handle, int sockfd)
+esp_err_t httpd_sess_update_lru_counter(httpd_handle_t handle, int sockfd)
 {
     if (handle == NULL) {
         return ESP_ERR_INVALID_ARG;
@@ -312,7 +318,7 @@ esp_err_t httpd_sess_update_timestamp(httpd_handle_t handle, int sockfd)
     int i;
     for (i = 0; i < hd->config.max_open_sockets; i++) {
         if (hd->hd_sd[i].fd == sockfd) {
-            hd->hd_sd[i].timestamp = httpd_os_get_timestamp();
+            hd->hd_sd[i].lru_counter = httpd_sess_get_lru_counter();
             return ESP_OK;
         }
     }
@@ -321,7 +327,7 @@ esp_err_t httpd_sess_update_timestamp(httpd_handle_t handle, int sockfd)
 
 esp_err_t httpd_sess_close_lru(struct httpd_data *hd)
 {
-    int64_t timestamp = INT64_MAX;
+    uint64_t lru_counter = UINT64_MAX;
     int lru_fd = -1;
     int i;
     for (i = 0; i < hd->config.max_open_sockets; i++) {
@@ -332,8 +338,8 @@ esp_err_t httpd_sess_close_lru(struct httpd_data *hd)
         if (hd->hd_sd[i].fd == -1) {
             return ESP_OK;
         }
-        if (hd->hd_sd[i].timestamp < timestamp) {
-            timestamp = hd->hd_sd[i].timestamp;
+        if (hd->hd_sd[i].lru_counter < lru_counter) {
+            lru_counter = hd->hd_sd[i].lru_counter;
             lru_fd = hd->hd_sd[i].fd;
         }
     }

components/esp_http_server/src/port/esp32/osal.h

@@ -52,11 +52,6 @@ static inline void httpd_os_thread_sleep(int msecs)
     vTaskDelay(msecs / portTICK_RATE_MS);
 }
 
-static inline int64_t httpd_os_get_timestamp()
-{
-    return esp_timer_get_time();
-}
-
 static inline othread_t httpd_os_thread_handle()
 {
     return xTaskGetCurrentTaskHandle();

components/freertos/include/freertos/portable.h

@@ -205,7 +205,7 @@ BaseType_t xPortInterruptedFromISRContext();
 /* Multi-core: get current core ID */
 static inline uint32_t IRAM_ATTR xPortGetCoreID() {
     int id;
-    __asm__ (
+    __asm__ __volatile__ (
         "rsr.prid %0\n"
         " extui %0,%0,13,1"
         :"=r"(id));

components/freertos/queue.c

@@ -338,6 +338,7 @@ Queue_t * const pxQueue = ( Queue_t * ) xQueue;
 			the real queue and semaphore structures. */
 			volatile size_t xSize = sizeof( StaticQueue_t );
 			configASSERT( xSize == sizeof( Queue_t ) );
+			( void ) xSize; /* Keeps lint quiet when configASSERT() is not defined. */
 		}
 		#endif /* configASSERT_DEFINED */
 

components/freertos/timers.c

@@ -347,6 +347,7 @@ BaseType_t xReturn = pdFAIL;
 			structures. */
 			volatile size_t xSize = sizeof( StaticTimer_t );
 			configASSERT( xSize == sizeof( Timer_t ) );
+			( void ) xSize; /* Keeps lint quiet when configASSERT() is not defined. */
 		}
 		#endif /* configASSERT_DEFINED */
 

components/mdns/mdns.c

@@ -1452,20 +1452,13 @@ static void _mdns_pcb_send_bye(tcpip_adapter_if_t tcpip_if, mdns_ip_protocol_t i
 }
 
 /**
- * @brief  Send probe for particular services on particular PCB
+ * @brief  Send probe for additional services on particular PCB
  */
-static void _mdns_init_pcb_probe(tcpip_adapter_if_t tcpip_if, mdns_ip_protocol_t ip_protocol, mdns_srv_item_t ** services, size_t len, bool probe_ip)
+static void _mdns_init_pcb_probe_new_service(tcpip_adapter_if_t tcpip_if, mdns_ip_protocol_t ip_protocol, mdns_srv_item_t ** services, size_t len, bool probe_ip)
 {
     mdns_pcb_t * pcb = &_mdns_server->interfaces[tcpip_if].pcbs[ip_protocol];
     size_t services_final_len = len;
 
-    _mdns_clear_pcb_tx_queue_head(tcpip_if, ip_protocol);
-
-    if (_str_null_or_empty(_mdns_server->hostname)) {
-        pcb->state = PCB_RUNNING;
-        return;
-    }
-
     if (PCB_STATE_IS_PROBING(pcb)) {
         services_final_len += pcb->probe_services_len;
     }
@@ -1510,6 +1503,50 @@ static void _mdns_init_pcb_probe(tcpip_adapter_if_t tcpip_if, mdns_ip_protocol_t
     pcb->state = PCB_PROBE_1;
 }
 
+/**
+ * @brief  Send probe for particular services on particular PCB
+ *
+ * Tests possible duplication on probing service structure and probes only for new entries.
+ * - If pcb probing then add only non-probing services and restarts probing
+ * - If pcb not probing, run probing for all specified services
+ */
+static void _mdns_init_pcb_probe(tcpip_adapter_if_t tcpip_if, mdns_ip_protocol_t ip_protocol, mdns_srv_item_t ** services, size_t len, bool probe_ip)
+{
+    mdns_pcb_t * pcb = &_mdns_server->interfaces[tcpip_if].pcbs[ip_protocol];
+
+    _mdns_clear_pcb_tx_queue_head(tcpip_if, ip_protocol);
+
+    if (_str_null_or_empty(_mdns_server->hostname)) {
+        pcb->state = PCB_RUNNING;
+        return;
+    }
+
+    if (PCB_STATE_IS_PROBING(pcb)) {
+        // Looking for already probing services to resolve duplications
+        mdns_srv_item_t * new_probe_services[len];
+        int new_probe_service_len = 0;
+        bool found;
+        for (int j=0; j < len; ++j) {
+            found = false;
+            for (int i=0; i < pcb->probe_services_len; ++i) {
+                if (pcb->probe_services[i] == services[j]) {
+                    found = true;
+                    break;
+                }
+            }
+            if (!found) {
+                new_probe_services[new_probe_service_len++] = services[j];
+            }
+        }
+        // init probing for newly added services
+        _mdns_init_pcb_probe_new_service(tcpip_if, ip_protocol,
+                                         new_probe_service_len?new_probe_services:NULL, new_probe_service_len, probe_ip);
+    } else {
+        // not probing, so init for all services
+        _mdns_init_pcb_probe_new_service(tcpip_if, ip_protocol, services, len, probe_ip);
+    }
+}
+
 /**
  * @brief  Restart the responder on particular PCB
  */
@@ -1863,6 +1900,9 @@ static void _mdns_dealloc_scheduled_service_answers(mdns_out_answer_t ** destina
  */
 static void _mdns_remove_scheduled_service_packets(mdns_service_t * service)
 {
+    if (!service) {
+        return;
+    }
     mdns_tx_packet_t * p = NULL;
     mdns_tx_packet_t * q = _mdns_server->tx_queue_head;
     while (q) {
@@ -1951,7 +1991,6 @@ static void _mdns_free_service(mdns_service_t * service)
     if (!service) {
         return;
     }
-    _mdns_remove_scheduled_service_packets(service);
     free((char *)service->instance);
     free((char *)service->service);
     free((char *)service->proto);
@@ -3074,7 +3113,7 @@ static void _mdns_search_free(mdns_search_once_t * search)
     free(search->instance);
     free(search->service);
     free(search->proto);
-    vSemaphoreDelete(search->lock);
+    vSemaphoreDelete(search->done_semaphore);
     free(search);
 }
 
@@ -3090,8 +3129,8 @@ static mdns_search_once_t * _mdns_search_init(const char * name, const char * se
     }
     memset(search, 0, sizeof(mdns_search_once_t));
 
-    search->lock = xSemaphoreCreateMutex();
-    if (!search->lock) {
+    search->done_semaphore = xSemaphoreCreateBinary();
+    if (!search->done_semaphore) {
         free(search);
         return NULL;
     }
@@ -3130,8 +3169,6 @@ static mdns_search_once_t * _mdns_search_init(const char * name, const char * se
     search->started_at = xTaskGetTickCount() * portTICK_PERIOD_MS;
     search->next = NULL;
 
-    xSemaphoreTake(search->lock, 0);
-
     return search;
 }
 
@@ -3142,7 +3179,7 @@ static void _mdns_search_finish(mdns_search_once_t * search)
 {
     search->state = SEARCH_OFF;
     queueDetach(mdns_search_once_t, _mdns_server->search_once, search);
-    xSemaphoreGive(search->lock);
+    xSemaphoreGive(search->done_semaphore);
 }
 
 /**
@@ -3783,6 +3820,7 @@ static void _mdns_execute_action(mdns_action_t * action)
         if (_mdns_server->services == action->data.srv_del.service) {
             _mdns_server->services = a->next;
             _mdns_send_bye(&a, 1, false);
+            _mdns_remove_scheduled_service_packets(a->service);
             _mdns_free_service(a->service);
             free(a);
         } else {
@@ -3793,6 +3831,7 @@ static void _mdns_execute_action(mdns_action_t * action)
                 mdns_srv_item_t * b = a->next;
                 a->next = a->next->next;
                 _mdns_send_bye(&b, 1, false);
+                _mdns_remove_scheduled_service_packets(b->service);
                 _mdns_free_service(b->service);
                 free(b);
             }
@@ -3806,6 +3845,7 @@ static void _mdns_execute_action(mdns_action_t * action)
         while (a) {
             mdns_srv_item_t * s = a;
             a = a->next;
+            _mdns_remove_scheduled_service_packets(s->service);
             _mdns_free_service(s->service);
             free(s);
         }
@@ -3821,7 +3861,17 @@ static void _mdns_execute_action(mdns_action_t * action)
         _mdns_search_finish(action->data.search_add.search);
         break;
     case ACTION_TX_HANDLE:
-        _mdns_tx_handle_packet(action->data.tx_handle.packet);
+        {
+            mdns_tx_packet_t * p = _mdns_server->tx_queue_head;
+            // packet to be handled should be at tx head, but must be consistent with the one pushed to action queue
+            if (p && p==action->data.tx_handle.packet && p->queued) {
+                p->queued = false; // clearing, as the packet might be reused (pushed and transmitted again)
+                _mdns_server->tx_queue_head = p->next;
+                _mdns_tx_handle_packet(p);
+            } else {
+                ESP_LOGD(TAG, "Skipping transmit of an unexpected packet!");
+            }
+        }
         break;
     case ACTION_RX_HANDLE:
         mdns_parse_packet(action->data.rx_handle.packet);
@@ -3858,6 +3908,10 @@ static esp_err_t _mdns_send_search_action(mdns_action_type_t type, mdns_search_o
 
 /**
  * @brief  Called from timer task to run mDNS responder
+ *
+ * periodically checks first unqueued packet (from tx head).
+ * if it is scheduled to be transmitted, then pushes the packet to action queue to be handled.
+ *
  */
 static void _mdns_scheduler_run()
 {
@@ -3865,6 +3919,10 @@ static void _mdns_scheduler_run()
     mdns_tx_packet_t * p = _mdns_server->tx_queue_head;
     mdns_action_t * action = NULL;
 
+    // find first unqueued packet
+    while (p && p->queued) {
+        p = p->next;
+    }
     if (!p) {
         MDNS_SERVICE_UNLOCK();
         return;
@@ -3872,12 +3930,12 @@ static void _mdns_scheduler_run()
     if ((int32_t)(p->send_at - (xTaskGetTickCount() * portTICK_PERIOD_MS)) < 0) {
         action = (mdns_action_t *)malloc(sizeof(mdns_action_t));
         if (action) {
-            _mdns_server->tx_queue_head = p->next;
             action->type = ACTION_TX_HANDLE;
             action->data.tx_handle.packet = p;
+            p->queued = true;
             if (xQueueSend(_mdns_server->action_queue, &action, (portTickType)0) != pdPASS) {
                 free(action);
-                _mdns_server->tx_queue_head = p;
+                p->queued = false;
             }
         } else {
             HOOK_MALLOC_FAILED;
@@ -4017,9 +4075,7 @@ static esp_err_t _mdns_service_task_start()
  */
 static esp_err_t _mdns_service_task_stop()
 {
-    MDNS_SERVICE_LOCK();
     _mdns_stop_timer();
-    MDNS_SERVICE_UNLOCK();
     if (_mdns_service_task_handle) {
         mdns_action_t action;
         mdns_action_t * a = &action;
@@ -4088,12 +4144,6 @@ esp_err_t mdns_init()
         goto free_lock;
     }
 
-    if (_mdns_service_task_start()) {
-        //service start failed!
-        err = ESP_FAIL;
-        goto free_all;
-    }
-
     uint8_t i;
     ip6_addr_t tmp_addr6;
     tcpip_adapter_ip_info_t if_ip_info;
@@ -4107,9 +4157,19 @@ esp_err_t mdns_init()
         }
     }
 
+    if (_mdns_service_task_start()) {
+        //service start failed!
+        err = ESP_FAIL;
+        goto free_all_and_disable_pcbs;
+    }
+
     return ESP_OK;
 
-free_all:
+free_all_and_disable_pcbs:
+    for (i=0; i<TCPIP_ADAPTER_IF_MAX; i++) {
+        _mdns_disable_pcb(i, MDNS_IP_PROTOCOL_V6);
+        _mdns_disable_pcb(i, MDNS_IP_PROTOCOL_V4);
+    }
     vQueueDelete(_mdns_server->action_queue);
 free_lock:
     vSemaphoreDelete(_mdns_server->lock);
@@ -4148,7 +4208,7 @@ void mdns_free()
         free(h->instance);
         free(h->service);
         free(h->proto);
-        vSemaphoreDelete(h->lock);
+        vSemaphoreDelete(h->done_semaphore);
         if (h->result) {
             mdns_query_results_free(h->result);
         }
@@ -4543,7 +4603,7 @@ esp_err_t mdns_query(const char * name, const char * service, const char * proto
         _mdns_search_free(search);
         return ESP_ERR_NO_MEM;
     }
-    xSemaphoreTake(search->lock, portMAX_DELAY);
+    xSemaphoreTake(search->done_semaphore, portMAX_DELAY);
 
     *results = search->result;
     _mdns_search_free(search);

components/mdns/private_include/mdns_private.h

@@ -115,9 +115,6 @@
 #define PCB_STATE_IS_ANNOUNCING(s) (s->state > PCB_PROBE_3 && s->state < PCB_RUNNING)
 #define PCB_STATE_IS_RUNNING(s) (s->state == PCB_RUNNING)
 
-#define MDNS_SEARCH_LOCK()      xSemaphoreTake(_mdns_server->search.lock, portMAX_DELAY)
-#define MDNS_SEARCH_UNLOCK()    xSemaphoreGive(_mdns_server->search.lock)
-
 #ifndef HOOK_MALLOC_FAILED
 #define HOOK_MALLOC_FAILED  ESP_LOGE(TAG, "Cannot allocate memory (line: %d, free heap: %d bytes)", __LINE__, esp_get_free_heap_size());
 #endif
@@ -292,6 +289,7 @@ typedef struct mdns_tx_packet_s {
     mdns_out_answer_t * answers;
     mdns_out_answer_t * servers;
     mdns_out_answer_t * additional;
+    bool queued;
 } mdns_tx_packet_t;
 
 typedef struct {
@@ -318,7 +316,7 @@ typedef struct mdns_search_once_s {
     uint32_t started_at;
     uint32_t sent_at;
     uint32_t timeout;
-    SemaphoreHandle_t lock;
+    SemaphoreHandle_t done_semaphore;
     uint16_t type;
     uint8_t max_results;
     uint8_t num_results;

components/nvs_flash/src/nvs_item_hash_list.cpp

@@ -64,15 +64,22 @@ void HashList::erase(size_t index, bool itemShouldExist)
 {
     for (auto it = mBlockList.begin(); it != mBlockList.end();) {
         bool haveEntries = false;
+        bool foundIndex = false;
         for (size_t i = 0; i < it->mCount; ++i) {
             if (it->mNodes[i].mIndex == index) {
                 it->mNodes[i].mIndex = 0xff;
-                return;
+                foundIndex = true;
+                /* found the item and removed it */
             }
             if (it->mNodes[i].mIndex != 0xff) {
                 haveEntries = true;
             }
+            if (haveEntries && foundIndex) {
+                /* item was found, and HashListBlock still has some items */
+                return;
+            }
         }
+        /* no items left in HashListBlock, can remove */
         if (!haveEntries) {
             auto tmp = it;
             ++it;
@@ -81,6 +88,10 @@ void HashList::erase(size_t index, bool itemShouldExist)
         } else {
             ++it;
         }
+        if (foundIndex) {
+            /* item was found and empty HashListBlock was removed */
+            return;
+        }
     }
     if (itemShouldExist) {
         assert(false && "item should have been present in cache");

components/nvs_flash/test_nvs_host/test_nvs.cpp

@@ -284,6 +284,47 @@ TEST_CASE("Page handles invalid CRC of variable length items", "[nvs][cur]")
     }
 }
 
+class HashListTestHelper : public HashList
+{
+    public:
+        size_t getBlockCount()
+        {
+            return mBlockList.size();
+        }
+};
+
+TEST_CASE("HashList is cleaned up as soon as items are erased", "[nvs]")
+{
+    HashListTestHelper hashlist;
+    // Add items
+    const size_t count = 128;
+    for (size_t i = 0; i < count; ++i) {
+        char key[16];
+        snprintf(key, sizeof(key), "i%ld", (long int)i);
+        Item item(1, ItemType::U32, 1, key);
+        hashlist.insert(item, i);
+    }
+    INFO("Added " << count << " items, " << hashlist.getBlockCount() << " blocks");
+    // Remove them in reverse order
+    for (size_t i = count; i > 0; --i) {
+        hashlist.erase(i - 1, true);
+    }
+    CHECK(hashlist.getBlockCount() == 0);
+    // Add again
+    for (size_t i = 0; i < count; ++i) {
+        char key[16];
+        snprintf(key, sizeof(key), "i%ld", (long int)i);
+        Item item(1, ItemType::U32, 1, key);
+        hashlist.insert(item, i);
+    }
+    INFO("Added " << count << " items, " << hashlist.getBlockCount() << " blocks");
+    // Remove them in the same order
+    for (size_t i = 0; i < count; ++i) {
+        hashlist.erase(i, true);
+    }
+    CHECK(hashlist.getBlockCount() == 0);
+}
+
 TEST_CASE("can init PageManager in empty flash", "[nvs]")
 {
     SpiFlashEmulator emu(4);
@@ -1716,6 +1757,28 @@ TEST_CASE("Check that orphaned blobs are erased during init", "[nvs]")
     TEST_ESP_OK(storage.writeItem(1, ItemType::BLOB, "key3", blob, sizeof(blob)));
 }
 
+TEST_CASE("nvs blob fragmentation test", "[nvs]")
+{
+    SpiFlashEmulator emu(4);
+    TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 4) );
+    const size_t BLOB_SIZE = 3500;
+    uint8_t *blob = (uint8_t*) malloc(BLOB_SIZE);
+    CHECK(blob != NULL);
+    memset(blob, 0xEE, BLOB_SIZE);
+    const uint32_t magic = 0xff33eaeb;
+    nvs_handle h;
+    TEST_ESP_OK( nvs_open("blob_tests", NVS_READWRITE, &h) );
+    for (int i = 0; i < 128; i++) {
+        INFO("Iteration " << i << "...\n");
+        TEST_ESP_OK( nvs_set_u32(h, "magic", magic) );
+        TEST_ESP_OK( nvs_set_blob(h, "blob", blob, BLOB_SIZE) );
+        char seq_buf[16];
+        sprintf(seq_buf, "seq%d", i);
+        TEST_ESP_OK( nvs_set_u32(h, seq_buf, i) );
+    }
+    free(blob);
+}
+
 TEST_CASE("nvs code handles errors properly when partition is near to full", "[nvs]")
 {
     const size_t blob_size = Page::CHUNK_MAX_SIZE * 0.3 ;

components/protocomm/src/transports/protocomm_ble.c

@@ -110,25 +110,22 @@ static void transport_simple_ble_read(esp_gatts_cb_event_t event, esp_gatt_if_t
         ESP_LOGD(TAG, "Subsequent read request for attr value");
     }
 
-    esp_gatt_rsp_t *gatt_rsp = (esp_gatt_rsp_t *) malloc(sizeof(esp_gatt_rsp_t));
-    if (gatt_rsp != NULL) {
-        gatt_rsp->attr_value.len = MIN(read_len, (protoble_internal->gatt_mtu - 1));
-        if (read_len && read_buf) {
-            memcpy(gatt_rsp->attr_value.value,
-                   read_buf + param->read.offset,
-                   gatt_rsp->attr_value.len);
-        }
-        read_len -= gatt_rsp->attr_value.len;
-    } else {
-        ESP_LOGE(TAG, "%s, malloc failed", __func__);
-        return;
+    esp_gatt_rsp_t gatt_rsp = {0};
+    gatt_rsp.attr_value.len = MIN(read_len, (protoble_internal->gatt_mtu - 1));
+    gatt_rsp.attr_value.handle = param->read.handle;
+    gatt_rsp.attr_value.offset = param->read.offset;
+    gatt_rsp.attr_value.auth_req = ESP_GATT_AUTH_REQ_NONE;
+    if (gatt_rsp.attr_value.len && read_buf) {
+        memcpy(gatt_rsp.attr_value.value,
+                read_buf + param->read.offset,
+                gatt_rsp.attr_value.len);
     }
+    read_len -= gatt_rsp.attr_value.len;
     esp_err_t err = esp_ble_gatts_send_response(gatts_if, param->read.conn_id,
-                                                param->read.trans_id, status, gatt_rsp);
+                                                param->read.trans_id, status, &gatt_rsp);
     if (err != ESP_OK) {
         ESP_LOGE(TAG, "Send response error in read");
     }
-    free(gatt_rsp);
 }
 
 static esp_err_t prepare_write_event_env(esp_gatt_if_t gatts_if,
@@ -157,22 +154,19 @@ static esp_err_t prepare_write_event_env(esp_gatt_if_t gatts_if,
     prepare_write_env.prepare_len += param->write.len;
     prepare_write_env.handle = param->write.handle;
     if (param->write.need_rsp) {
-        esp_gatt_rsp_t *gatt_rsp = (esp_gatt_rsp_t *) malloc(sizeof(esp_gatt_rsp_t));
-        if (gatt_rsp != NULL) {
-            gatt_rsp->attr_value.len = param->write.len;
-            gatt_rsp->attr_value.handle = param->write.handle;
-            gatt_rsp->attr_value.offset = param->write.offset;
-            gatt_rsp->attr_value.auth_req = ESP_GATT_AUTH_REQ_NONE;
-            memcpy(gatt_rsp->attr_value.value, param->write.value, param->write.len);
-            esp_err_t response_err = esp_ble_gatts_send_response(gatts_if, param->write.conn_id,
-                                                                 param->write.trans_id, status,
-                                                                 gatt_rsp);
-            if (response_err != ESP_OK) {
-                ESP_LOGE(TAG, "Send response error in prep write");
-            }
-            free(gatt_rsp);
-        } else {
-            ESP_LOGE(TAG, "%s, malloc failed", __func__);
+        esp_gatt_rsp_t gatt_rsp = {0};
+        gatt_rsp.attr_value.len = param->write.len;
+        gatt_rsp.attr_value.handle = param->write.handle;
+        gatt_rsp.attr_value.offset = param->write.offset;
+        gatt_rsp.attr_value.auth_req = ESP_GATT_AUTH_REQ_NONE;
+        if (gatt_rsp.attr_value.len && param->write.value) {
+            memcpy(gatt_rsp.attr_value.value, param->write.value, param->write.len);
+        }
+        esp_err_t response_err = esp_ble_gatts_send_response(gatts_if, param->write.conn_id,
+                param->write.trans_id, status,
+                &gatt_rsp);
+        if (response_err != ESP_OK) {
+            ESP_LOGE(TAG, "Send response error in prep write");
         }
     }
     if (status != ESP_GATT_OK) {

components/soc/esp32/include/soc/dport_reg.h

@@ -179,6 +179,9 @@
 #define DPORT_CPUPERIOD_SEL_M  ((DPORT_CPUPERIOD_SEL_V)<<(DPORT_CPUPERIOD_SEL_S))
 #define DPORT_CPUPERIOD_SEL_V  0x3
 #define DPORT_CPUPERIOD_SEL_S  0
+#define DPORT_CPUPERIOD_SEL_80		0
+#define DPORT_CPUPERIOD_SEL_160		1
+#define DPORT_CPUPERIOD_SEL_240		2
 
 #define DPORT_PRO_CACHE_CTRL_REG          (DR_REG_DPORT_BASE + 0x040)
 /* DPORT_PRO_DRAM_HL : R/W ;bitpos:[16] ;default: 1'b0 ; */

components/soc/esp32/rtc_clk.c

@@ -395,7 +395,6 @@ void rtc_clk_cpu_freq_to_xtal(int freq, int div)
     REG_WRITE(APB_CTRL_XTAL_TICK_CONF_REG, freq * MHZ / REF_CLK_FREQ - 1);
     /* switch clock source */
     REG_SET_FIELD(RTC_CNTL_CLK_CONF_REG, RTC_CNTL_SOC_CLK_SEL, RTC_CNTL_SOC_CLK_SEL_XTL);
-    DPORT_REG_WRITE(DPORT_CPU_PER_CONF_REG, 0); /* clear DPORT_CPUPERIOD_SEL */
     rtc_clk_apb_freq_update(freq * MHZ);
     /* lower the voltage */
     if (freq <= 2) {
@@ -411,7 +410,6 @@ static void rtc_clk_cpu_freq_to_8m()
     REG_SET_FIELD(RTC_CNTL_REG, RTC_CNTL_DIG_DBIAS_WAK, DIG_DBIAS_XTAL);
     REG_SET_FIELD(APB_CTRL_SYSCLK_CONF_REG, APB_CTRL_PRE_DIV_CNT, 0);
     REG_SET_FIELD(RTC_CNTL_CLK_CONF_REG, RTC_CNTL_SOC_CLK_SEL, RTC_CNTL_SOC_CLK_SEL_8M);
-    DPORT_REG_WRITE(DPORT_CPU_PER_CONF_REG, 0); // clear DPORT_CPUPERIOD_SEL
     rtc_clk_apb_freq_update(RTC_FAST_CLK_FREQ_8M);
 }
 
@@ -452,14 +450,14 @@ static void rtc_clk_bbpll_enable()
 static void rtc_clk_cpu_freq_to_pll_mhz(int cpu_freq_mhz)
 {
     int dbias = DIG_DBIAS_80M_160M;
-    int per_conf = 0;
+    int per_conf = DPORT_CPUPERIOD_SEL_80;
     if (cpu_freq_mhz == 80) {
         /* nothing to do */
     } else if (cpu_freq_mhz == 160) {
-        per_conf = 1;
+        per_conf = DPORT_CPUPERIOD_SEL_160;
     } else if (cpu_freq_mhz == 240) {
         dbias = DIG_DBIAS_240M;
-        per_conf = 2;
+        per_conf = DPORT_CPUPERIOD_SEL_240;
     } else {
         SOC_LOGE(TAG, "invalid frequency");
         abort();
@@ -687,15 +685,15 @@ void rtc_clk_cpu_freq_get_config(rtc_cpu_freq_config_t* out_config)
         case RTC_CNTL_SOC_CLK_SEL_PLL: {
             source = RTC_CPU_FREQ_SRC_PLL;
             uint32_t cpuperiod_sel = DPORT_REG_GET_FIELD(DPORT_CPU_PER_CONF_REG, DPORT_CPUPERIOD_SEL);
-            if (cpuperiod_sel == 0) {
+            if (cpuperiod_sel == DPORT_CPUPERIOD_SEL_80) {
                 source_freq_mhz = RTC_PLL_FREQ_320M;
                 div = 4;
                 freq_mhz = 80;
-            } else if (cpuperiod_sel == 1) {
+            } else if (cpuperiod_sel == DPORT_CPUPERIOD_SEL_160) {
                 source_freq_mhz = RTC_PLL_FREQ_320M;
                 div = 2;
                 freq_mhz = 160;
-            } else if (cpuperiod_sel == 2) {
+            } else if (cpuperiod_sel == DPORT_CPUPERIOD_SEL_240) {
                 source_freq_mhz = RTC_PLL_FREQ_480M;
                 div = 2;
                 freq_mhz = 240;

components/tcp_transport/transport_ws.c

@@ -191,7 +191,10 @@ static int ws_read(esp_transport_handle_t t, char *buffer, int len, int timeout_
     if ((poll_read = esp_transport_poll_read(ws->parent, timeout_ms)) <= 0) {
         return poll_read;
     }
-    if ((rlen = esp_transport_read(ws->parent, buffer, len, timeout_ms)) <= 0) {
+
+    // Receive and process header first (based on header size)
+    int header = 2;
+    if ((rlen = esp_transport_read(ws->parent, buffer, header, timeout_ms)) <= 0) {
         ESP_LOGE(TAG, "Error read data");
         return rlen;
     }
@@ -203,11 +206,20 @@ static int ws_read(esp_transport_handle_t t, char *buffer, int len, int timeout_
     ESP_LOGD(TAG, "Opcode: %d, mask: %d, len: %d\r\n", opcode, mask, payload_len);
     if (payload_len == 126) {
         // headerLen += 2;
+        if ((rlen = esp_transport_read(ws->parent, data_ptr, header, timeout_ms)) <= 0) {
+            ESP_LOGE(TAG, "Error read data");
+            return rlen;
+        }
         payload_len = data_ptr[0] << 8 | data_ptr[1];
         payload_len_buff = len - 4;
         data_ptr += 2;
     } else if (payload_len == 127) {
         // headerLen += 8;
+        header = 8;
+        if ((rlen = esp_transport_read(ws->parent, data_ptr, header, timeout_ms)) <= 0) {
+            ESP_LOGE(TAG, "Error read data");
+            return rlen;
+        }
 
         if (data_ptr[0] != 0 || data_ptr[1] != 0 || data_ptr[2] != 0 || data_ptr[3] != 0) {
             // really too big!
@@ -218,6 +230,11 @@ static int ws_read(esp_transport_handle_t t, char *buffer, int len, int timeout_
         data_ptr += 8;
         payload_len_buff = len - 10;
     }
+    // Then receive and process payload
+    if ((rlen = esp_transport_read(ws->parent, data_ptr, payload_len, timeout_ms)) <= 0) {
+        ESP_LOGE(TAG, "Error read data");
+        return rlen;
+    }
     if (payload_len > payload_len_buff) {
         ESP_LOGD(TAG, "Actual data received (%d) are longer than mqtt buffer (%d)", payload_len, payload_len_buff);
         payload_len = payload_len_buff;

docs/check_doc_warnings.sh

@@ -18,8 +18,7 @@ fi
 # the Sphinx warning log
 # (escape char removal from https://www.commandlinefu.com/commands/view/6141/remove-color-codes-special-characters-with-sed
 sed -r 's:\x1B\[[0-9;]*[mK]::g' sphinx-warning-log.txt | \
-    sed -E "s~${IDF_PATH}~\${IDF_PATH}~" | \
-    sed -E "s/:[0-9]+:/:line:/" > sphinx-warning-log-sanitized.txt
+    sed -E "s/.*\/(.*):[0-9]+:/\1:line:/" > sphinx-warning-log-sanitized.txt
 
 # diff sanitized warnings, ignoring lines which only appear in ../sphinx-known-warnings.txt
 

docs/en/api-guides/build-system-cmake.rst

@@ -720,6 +720,8 @@ the ESP-IDF build system entirely by using a CMake feature called ExternalProjec
 - The second set of commands adds a library target, which points to the "imported" library file built by the external system. Some properties need to be set in order to add include directories and tell CMake where this file is.
 - Finally, the generated library is added to `ADDITIONAL_MAKE_CLEAN_FILES`_. This means ``make clean`` will delete this library. (Note that the other object files from the build won't be deleted.)
 
+.. note:: When using an external build process with PSRAM, remember to add ``-mfix-esp32-psram-cache-issue`` to the C compiler arguments. See :ref:`CONFIG_SPIRAM_CACHE_WORKAROUND` for details of this flag.
+
 .. _ADDITIONAL_MAKE_CLEAN_FILES_note:
 
 ExternalProject dependencies, clean builds

docs/en/api-guides/build-system.rst

@@ -586,6 +586,7 @@ $(COMPONENT_LIBRARY) for the project make process to link into the app binary.
 (Actually, even this is not strictly necessary - if the COMPONENT_ADD_LDFLAGS variable
 is overridden then the component can instruct the linker to link other binaries instead.)
 
+.. note:: When using an external build process with PSRAM, remember to add ``-mfix-esp32-psram-cache-issue`` to the C compiler arguments. See :ref:`CONFIG_SPIRAM_CACHE_WORKAROUND` for details of this flag.
 
 .. _esp-idf-template: https://github.com/espressif/esp-idf-template
 .. _GNU Make Manual: https://www.gnu.org/software/make/manual/make.html

docs/requirements.txt

@@ -1,12 +1,12 @@
 # This is a list of python packages used to generate documentation. This file is used with pip:
 # pip install --user -r requirements.txt
 #
-sphinx==1.6.5
+sphinx>=1.8.4
+breathe>=4.11.1
 sphinx-rtd-theme
-breathe==4.7.3
-sphinxcontrib-blockdiag==1.5.3
-sphinxcontrib-seqdiag==0.8.5
-sphinxcontrib-actdiag==0.8.5
-sphinxcontrib-nwdiag==0.9.5
+sphinxcontrib-blockdiag>=1.5.5
+sphinxcontrib-seqdiag>=0.8.5
+sphinxcontrib-actdiag>=0.8.5
+sphinxcontrib-nwdiag>=0.9.5
 recommonmark
-future>=0.16.0 # for ../tools/gen_esp_err_to_name.py
+future>=0.16.0 # for ../tools/gen_esp_err_to_name.py

docs/sphinx-known-warnings.txt

@@ -2,7 +2,7 @@
 #
 # Build will fail if sphinx-warning-log.txt contains any lines
 # which are not in this file. Lines are pre-sanitized by
-# check_doc_warnings.sh to remove formatting, paths, line numbers.
+# check_doc_warnings.sh to remove formatting, paths and line numbers.
 #
 # Warnings in this file must be in the same overall order as the log file.
 #
@@ -10,9 +10,74 @@
 #
 # Sphinx known issue https://github.com/sphinx-doc/sphinx/issues/2683
 #
-_build/inc/esp_a2dp_api.inc:line: WARNING: Invalid definition: Expected identifier in nested name. [error at 21]
+# Note: warnings below will be gone after upgrade
+#       to the following package==version
+#
+#       sphinx==1.8.4
+#       breathe==4.11.1
+#
+esp_a2dp_api.inc:line: WARNING: Invalid definition: Expected identifier in nested name. [error at 21]
   union esp_a2d_mcc_t::@1  esp_a2d_mcc_t::cie
   ---------------------^
-_build/inc/esp_bt_defs.inc:line: WARNING: Invalid definition: Expected identifier in nested name. [error at 21]
+esp_bt_defs.inc:line: WARNING: Invalid definition: Expected identifier in nested name. [error at 21]
   union esp_bt_uuid_t::@0  esp_bt_uuid_t::uuid
   ---------------------^
+
+#
+# Breathe known issue: https://github.com/michaeljones/breathe/issues/405
+# Sphinx known issue: https://github.com/sphinx-doc/sphinx/pull/5901
+#
+# Note: warnings below have been identified after upgrade
+#       to the following package==version
+#
+#       sphinx==1.8.4
+#       breathe==4.11.1
+#
+ulp-cmake.rst:line: WARNING: Duplicate declaration, esp_err_t ulp_load_binary(uint32_t load_addr, const uint8_t * program_binary, size_t program_size)
+ulp-cmake.rst:line: WARNING: Duplicate declaration, esp_err_t ulp_run(uint32_t entry_point)
+ulp-cmake.rst:line: WARNING: Duplicate declaration, esp_err_t ulp_set_wakeup_period(size_t period_index, uint32_t period_us)
+README.rst:line: WARNING: Duplicate declaration, esp_err_t ulp_run(uint32_t entry_point)
+#
+# Issue present only when building on msys2 / mingw32 START >>>
+#
+esp_spp_api.inc:line: WARNING: Error in type declaration.
+If typedef-like declaration:
+  Type must be either just a name or a typedef-like declaration.
+  If just a name:
+    Error in declarator or parameters and qualifiers
+    Invalid definition: Expected identifier in nested name, got keyword: void [error at 4]
+      void() esp_spp_cb_t(esp_spp_cb_event_t event, esp_spp_cb_param_t *param)
+      ----^
+  If typedef-like declaration:
+    Error in declarator
+    If pointer to member declarator:
+      Invalid definition: Expected identifier in nested name. [error at 4]
+        void() esp_spp_cb_t(esp_spp_cb_event_t event, esp_spp_cb_param_t *param)
+        ----^
+    If declId, parameters, and qualifiers:
+      Invalid definition: Expected identifier in nested name. [error at 4]
+        void() esp_spp_cb_t(esp_spp_cb_event_t event, esp_spp_cb_param_t *param)
+        ----^
+    If parenthesis in noptr-declarator:
+      Error in declarator or parameters and qualifiers
+      If pointer to member declarator:
+        Invalid definition: Expected identifier in nested name. [error at 5]
+          void() esp_spp_cb_t(esp_spp_cb_event_t event, esp_spp_cb_param_t *param)
+          -----^
+      If declarator-id:
+        Invalid definition: Expected identifier in nested name. [error at 5]
+          void() esp_spp_cb_t(esp_spp_cb_event_t event, esp_spp_cb_param_t *param)
+          -----^
+If type alias or template alias:
+  Invalid definition: Expected identifier in nested name, got keyword: void [error at 4]
+    void() esp_spp_cb_t(esp_spp_cb_event_t event, esp_spp_cb_param_t *param)
+    ----^
+#
+# Issue present only when building on msys2 / mingw32 END <<<
+#
+spi_master.inc:line: WARNING: Duplicate declaration, struct spi_transaction_t spi_transaction_t
+spi_slave.inc:line: WARNING: Duplicate declaration, struct spi_slave_transaction_t spi_slave_transaction_t
+wear-levelling.rst:line: WARNING: Duplicate declaration, bool esp_vfs_fat_mount_config_t::format_if_mount_failed
+wear-levelling.rst:line: WARNING: Duplicate declaration, int esp_vfs_fat_mount_config_t::max_files
+wear-levelling.rst:line: WARNING: Duplicate declaration, size_t esp_vfs_fat_mount_config_t::allocation_unit_size
+wear-levelling.rst:line: WARNING: Duplicate declaration, esp_vfs_fat_mount_config_t

examples/ethernet/iperf/main/iperf_example_main.c

@@ -56,9 +56,8 @@ static void initialize_nvs()
 
 static void initialize_console()
 {
-    /* Disable buffering on stdin and stdout */
+    /* Disable buffering on stdin */
     setvbuf(stdin, NULL, _IONBF, 0);
-    setvbuf(stdout, NULL, _IONBF, 0);
 
     /* Minicom, screen, idf_monitor send CR when ENTER key is pressed */
     esp_vfs_dev_uart_set_rx_line_endings(ESP_LINE_ENDINGS_CR);

examples/peripherals/i2c/i2c_tools/main/i2ctools_example_main.c

@@ -56,9 +56,8 @@ static void initialize_nvs()
 
 static void initialize_console()
 {
-    /* Disable buffering on stdin and stdout */
+    /* Disable buffering on stdin */
     setvbuf(stdin, NULL, _IONBF, 0);
-    setvbuf(stdout, NULL, _IONBF, 0);
 
     /* Minicom, screen, idf_monitor send CR when ENTER key is pressed */
     esp_vfs_dev_uart_set_rx_line_endings(ESP_LINE_ENDINGS_CR);

examples/protocols/mqtt/ssl/main/app_main.c

@@ -116,6 +116,9 @@ static esp_err_t mqtt_event_handler(esp_mqtt_event_handle_t event)
         case MQTT_EVENT_ERROR:
             ESP_LOGI(TAG, "MQTT_EVENT_ERROR");
             break;
+        default:
+            ESP_LOGI(TAG, "Other event id:%d", event->event_id);
+            break;
     }
     return ESP_OK;
 }

examples/protocols/mqtt/ssl_mutual_auth/main/app_main.c

@@ -114,6 +114,9 @@ static esp_err_t mqtt_event_handler(esp_mqtt_event_handle_t event)
         case MQTT_EVENT_ERROR:
             ESP_LOGI(TAG, "MQTT_EVENT_ERROR");
             break;
+        default:
+            ESP_LOGI(TAG, "Other event id:%d", event->event_id);
+            break;
     }
     return ESP_OK;
 }

examples/protocols/mqtt/tcp/main/app_main.c

@@ -69,6 +69,9 @@ static esp_err_t mqtt_event_handler(esp_mqtt_event_handle_t event)
         case MQTT_EVENT_ERROR:
             ESP_LOGI(TAG, "MQTT_EVENT_ERROR");
             break;
+        default:
+            ESP_LOGI(TAG, "Other event id:%d", event->event_id);
+            break;
     }
     return ESP_OK;
 }

examples/protocols/mqtt/ws/main/app_main.c

@@ -66,6 +66,9 @@ static esp_err_t mqtt_event_handler(esp_mqtt_event_handle_t event)
         case MQTT_EVENT_ERROR:
             ESP_LOGI(TAG, "MQTT_EVENT_ERROR");
             break;
+        default:
+            ESP_LOGI(TAG, "Other event id:%d", event->event_id);
+            break;
     }
     return ESP_OK;
 }

examples/protocols/mqtt/wss/main/app_main.c

@@ -116,6 +116,9 @@ static esp_err_t mqtt_event_handler(esp_mqtt_event_handle_t event)
         case MQTT_EVENT_ERROR:
             ESP_LOGI(TAG, "MQTT_EVENT_ERROR");
             break;
+        default:
+            ESP_LOGI(TAG, "Other event id:%d", event->event_id);
+            break;
     }
     return ESP_OK;
 }

examples/protocols/sockets/tcp_client/main/tcp_client.c

@@ -147,6 +147,7 @@ static void tcp_client_task(void *pvParameters)
         int err = connect(sock, (struct sockaddr *)&destAddr, sizeof(destAddr));
         if (err != 0) {
             ESP_LOGE(TAG, "Socket unable to connect: errno %d", errno);
+            break;
         }
         ESP_LOGI(TAG, "Successfully connected");
 

examples/system/console/main/console_example_main.c

@@ -59,9 +59,8 @@ static void initialize_nvs()
 
 static void initialize_console()
 {
-    /* Disable buffering on stdin and stdout */
+    /* Disable buffering on stdin */
     setvbuf(stdin, NULL, _IONBF, 0);
-    setvbuf(stdout, NULL, _IONBF, 0);
 
     /* Minicom, screen, idf_monitor send CR when ENTER key is pressed */
     esp_vfs_dev_uart_set_rx_line_endings(ESP_LINE_ENDINGS_CR);

examples/system/ota/simple_ota_example/main/simple_ota_example.c

@@ -133,9 +133,10 @@ void app_main()
 {
     // Initialize NVS.
     esp_err_t err = nvs_flash_init();
-    if (err == ESP_ERR_NVS_NO_FREE_PAGES) {
-        // OTA app partition table has a smaller NVS partition size than the non-OTA
+    if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) {
+        // 1.OTA app partition table has a smaller NVS partition size than the non-OTA
         // partition table. This size mismatch may cause NVS initialization to fail.
+        // 2.NVS partition contains data in new format and cannot be recognized by this version of code.
         // If this happens, we erase NVS partition and initialize NVS again.
         ESP_ERROR_CHECK(nvs_flash_erase());
         err = nvs_flash_init();

examples/wifi/iperf/main/iperf_example_main.c

@@ -28,9 +28,8 @@
 
 static void initialize_console()
 {
-    /* Disable buffering on stdin and stdout */
+    /* Disable buffering on stdin */
     setvbuf(stdin, NULL, _IONBF, 0);
-    setvbuf(stdout, NULL, _IONBF, 0);
 
     /* Minicom, screen, idf_monitor send CR when ENTER key is pressed */
     esp_vfs_dev_uart_set_rx_line_endings(ESP_LINE_ENDINGS_CR);

examples/wifi/simple_sniffer/main/simple_sniffer_example_main.c

@@ -75,9 +75,8 @@ static void initialize_wifi()
 /* Initialize console component */
 static void initialize_console()
 {
-    /* Disable buffering on stdin and stdout */
+    /* Disable buffering on stdin */
     setvbuf(stdin, NULL, _IONBF, 0);
-    setvbuf(stdout, NULL, _IONBF, 0);
 
     /* Minicom, screen, idf_monitor send CR when ENTER key is pressed */
     esp_vfs_dev_uart_set_rx_line_endings(ESP_LINE_ENDINGS_CR);

make/project.mk

@@ -133,6 +133,9 @@ ifndef COMPONENT_DIRS
 EXTRA_COMPONENT_DIRS ?=
 COMPONENT_DIRS := $(PROJECT_PATH)/components $(EXTRA_COMPONENT_DIRS) $(IDF_PATH)/components $(PROJECT_PATH)/main
 endif
+# Make sure that every directory in the list is an absolute path without trailing slash.
+# This is necessary to split COMPONENT_DIRS into SINGLE_COMPONENT_DIRS and MULTI_COMPONENT_DIRS below. 
+COMPONENT_DIRS := $(foreach cd,$(COMPONENT_DIRS),$(abspath $(cd)))
 export COMPONENT_DIRS
 
 ifdef SRCDIRS
@@ -140,33 +143,55 @@ $(warning SRCDIRS variable is deprecated. These paths can be added to EXTRA_COMP
 COMPONENT_DIRS += $(abspath $(SRCDIRS))
 endif
 
-# The project Makefile can define a list of components, but if it does not do this we just take all available components
-# in the component dirs. A component is COMPONENT_DIRS directory, or immediate subdirectory,
+# List of component directories, i.e. directories which contain a component.mk file 
+SINGLE_COMPONENT_DIRS := $(abspath $(dir $(dir $(foreach cd,$(COMPONENT_DIRS),\
+                             $(wildcard $(cd)/component.mk)))))
+
+# List of components directories, i.e. directories which may contain components 
+MULTI_COMPONENT_DIRS := $(filter-out $(SINGLE_COMPONENT_DIRS),$(COMPONENT_DIRS))
+
+# The project Makefile can define a list of components, but if it does not do this
+# we just take all available components in the component dirs.
+# A component is COMPONENT_DIRS directory, or immediate subdirectory,
 # which contains a component.mk file.
 #
 # Use the "make list-components" target to debug this step.
 ifndef COMPONENTS
 # Find all component names. The component names are the same as the
 # directories they're in, so /bla/components/mycomponent/component.mk -> mycomponent.
-COMPONENTS := $(dir $(foreach cd,$(COMPONENT_DIRS),                           \
-					$(wildcard $(cd)/*/component.mk) $(wildcard $(cd)/component.mk) \
-				))
+# We need to do this for MULTI_COMPONENT_DIRS only, since SINGLE_COMPONENT_DIRS
+# are already known to contain component.mk.
+COMPONENTS := $(dir $(foreach cd,$(MULTI_COMPONENT_DIRS),$(wildcard $(cd)/*/component.mk))) \
+              $(SINGLE_COMPONENT_DIRS)
 COMPONENTS := $(sort $(foreach comp,$(COMPONENTS),$(lastword $(subst /, ,$(comp)))))
 endif
-# After a full manifest of component names is determined, subtract the ones explicitly omitted by the project Makefile.
+# After a full manifest of component names is determined, subtract the ones explicitly
+# omitted by the project Makefile.
+EXCLUDE_COMPONENTS ?=
 ifdef EXCLUDE_COMPONENTS
-COMPONENTS := $(filter-out $(subst ",,$(EXCLUDE_COMPONENTS)), $(COMPONENTS)) 
+COMPONENTS := $(filter-out $(subst ",,$(EXCLUDE_COMPONENTS)), $(COMPONENTS))
 # to keep syntax highlighters happy: "))
 endif
 export COMPONENTS
 
 # Resolve all of COMPONENTS into absolute paths in COMPONENT_PATHS.
+# For each entry in COMPONENT_DIRS:
+# - either this is directory with multiple components, in which case check that
+#   a subdirectory with component name exists, and it contains a component.mk file.
+# - or, this is a directory of a single component, in which case the name of this
+#   directory has to match the component name
 #
 # If a component name exists in multiple COMPONENT_DIRS, we take the first match.
 #
 # NOTE: These paths must be generated WITHOUT a trailing / so we
 # can use $(notdir x) to get the component name.
-COMPONENT_PATHS := $(foreach comp,$(COMPONENTS),$(firstword $(foreach cd,$(COMPONENT_DIRS),$(wildcard $(dir $(cd))$(comp) $(cd)/$(comp)))))
+COMPONENT_PATHS := $(foreach comp,$(COMPONENTS),\
+                        $(firstword $(foreach cd,$(COMPONENT_DIRS),\
+                            $(if $(findstring $(cd),$(MULTI_COMPONENT_DIRS)),\
+                                 $(abspath $(dir $(wildcard $(cd)/$(comp)/component.mk))),)\
+                            $(if $(findstring $(cd),$(SINGLE_COMPONENT_DIRS)),\
+                                 $(if $(filter $(comp),$(notdir $(cd))),$(cd),),)\
+                   )))
 export COMPONENT_PATHS
 
 TEST_COMPONENTS ?=
@@ -263,7 +288,7 @@ LDFLAGS ?= -nostdlib \
 #  before including project.mk. Default flags will be added before the ones provided in application Makefile.
 
 # CPPFLAGS used by C preprocessor
-# If any flags are defined in application Makefile, add them at the end. 
+# If any flags are defined in application Makefile, add them at the end.
 CPPFLAGS ?=
 EXTRA_CPPFLAGS ?=
 CPPFLAGS := -DESP_PLATFORM -D IDF_VER=\"$(IDF_VER)\" -MMD -MP $(CPPFLAGS) $(EXTRA_CPPFLAGS)
@@ -575,7 +600,7 @@ list-components:
 	$(info $(TEST_COMPONENTS_LIST))
 	$(info $(call dequote,$(SEPARATOR)))
 	$(info TEST_EXCLUDE_COMPONENTS (list of test excluded names))
-	$(info $(if $(EXCLUDE_COMPONENTS) || $(TEST_EXCLUDE_COMPONENTS),$(EXCLUDE_COMPONENTS) $(TEST_EXCLUDE_COMPONENTS),(none provided)))	
+	$(info $(if $(EXCLUDE_COMPONENTS) || $(TEST_EXCLUDE_COMPONENTS),$(EXCLUDE_COMPONENTS) $(TEST_EXCLUDE_COMPONENTS),(none provided)))
 	$(info $(call dequote,$(SEPARATOR)))
 	$(info COMPONENT_PATHS (paths to all components):)
 	$(foreach cp,$(COMPONENT_PATHS),$(info $(cp)))

tools/ci/test_build_system.sh

@@ -238,7 +238,7 @@ function run_tests()
     ( make 2>&1 | grep "does not fit in configured flash size 1MB" ) || failure "Build didn't fail with expected flash size failure message"
     mv sdkconfig.bak sdkconfig
 
-    print_status "sdkconfig should have contents both files: sdkconfig and sdkconfig.defaults"
+    print_status "sdkconfig should have contents of both files: sdkconfig and sdkconfig.defaults"
     make clean > /dev/null;
     rm -f sdkconfig.defaults;
     rm -f sdkconfig;
@@ -247,6 +247,35 @@ function run_tests()
     make defconfig > /dev/null;
     grep "CONFIG_PARTITION_TABLE_OFFSET=0x10000" sdkconfig || failure "The define from sdkconfig.defaults should be into sdkconfig"
     grep "CONFIG_PARTITION_TABLE_TWO_OTA=y" sdkconfig || failure "The define from sdkconfig should be into sdkconfig"
+    rm sdkconfig sdkconfig.defaults
+    make defconfig
+
+    print_status "Empty directory not treated as a component"
+    mkdir -p components/esp32
+    make || failure "Failed to build with empty esp32 directory in components"
+    rm -rf components
+
+    print_status "If a component directory is added to COMPONENT_DIRS, its subdirectories are not added"
+    mkdir -p main/test
+    touch main/test/component.mk
+    echo "#error This should not be built" > main/test/test.c
+    make || failure "COMPONENT_DIRS has added component subdirectory to the build"
+    rm -rf main/test
+
+    print_status "If a component directory is added to COMPONENT_DIRS, its sibling directories are not added"
+    mkdir -p mycomponents/mycomponent
+    touch mycomponents/mycomponent/component.mk
+    # first test by adding single component directory to EXTRA_COMPONENT_DIRS
+    mkdir -p mycomponents/esp32
+    touch mycomponents/esp32/component.mk
+    make EXTRA_COMPONENT_DIRS=$PWD/mycomponents/mycomponent || failure "EXTRA_COMPONENT_DIRS has added a sibling directory"
+    rm -rf mycomponents/esp32
+    # now the same thing, but add a components directory
+    mkdir -p esp32
+    touch esp32/component.mk
+    make EXTRA_COMPONENT_DIRS=$PWD/mycomponents || failure "EXTRA_COMPONENT_DIRS has added a sibling directory"
+    rm -rf esp32
+    rm -rf mycomponents
 
     print_status "All tests completed"
     if [ -n "${FAILURES}" ]; then

tools/ci/test_build_system_cmake.sh

@@ -223,6 +223,17 @@ function run_tests()
     grep "CONFIG_PARTITION_TABLE_OFFSET=0x10000" sdkconfig || failure "The define from sdkconfig.defaults should be into sdkconfig"
     grep "CONFIG_PARTITION_TABLE_TWO_OTA=y" sdkconfig || failure "The define from sdkconfig should be into sdkconfig"
 
+    print_status "Building a project with CMake and PSRAM workaround, all files compile with workaround"
+    cp sdkconfig sdkconfig.psram
+    rm -rf build
+    echo "CONFIG_SPIRAM_SUPPORT=y" >> sdkconfig.psram
+    echo "CONFIG_SPIRAM_CACHE_WORKAROUND=y" >> sdkconfig.psram
+    # note: we do 'reconfigure' here, as we just need to run cmake
+    idf.py reconfigure -D SDKCONFIG="`pwd`/sdkconfig.psram"
+    grep -q '"command"' build/compile_commands.json || failure "compile_commands.json missing or has no no 'commands' in it"
+    (grep '"command"' build/compile_commands.json | grep -v mfix-esp32-psram-cache-issue) && failure "All commands in compile_commands.json should use PSRAM cache workaround"
+    rm sdkconfig.psram
+
     print_status "All tests completed"
     if [ -n "${FAILURES}" ]; then
         echo "Some failures were detected:"

tools/cmake/idf_functions.cmake

@@ -65,14 +65,16 @@ function(idf_set_global_compiler_options)
         add_compile_options(-Og)
     endif()
 
-    add_c_compile_options(-std=gnu99)
+    # Note: the visual studio generator doesn't support this syntax
+    add_compile_options("$<$<COMPILE_LANGUAGE:C>:-std=gnu99>")
 
-    add_cxx_compile_options(-std=gnu++11 -fno-rtti)
+    add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:-std=gnu++11>")
+    add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:-fno-rtti>")
 
     if(CONFIG_CXX_EXCEPTIONS)
-        add_cxx_compile_options(-fexceptions)
+        add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:-fexceptions>")
     else()
-        add_cxx_compile_options(-fno-exceptions)
+        add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:-fno-exceptions>")
     endif()
 
     # Default compiler configuration
@@ -89,9 +91,7 @@ function(idf_set_global_compiler_options)
         -Wextra
         -Wno-unused-parameter
         -Wno-sign-compare)
-    add_c_compile_options(
-        -Wno-old-style-declaration
-        )
+    add_compile_options("$<$<COMPILE_LANGUAGE:C>:-Wno-old-style-declaration>")
 
     if(CONFIG_DISABLE_GCC8_WARNINGS)
         add_compile_options(
@@ -258,4 +258,4 @@ function(idf_get_git_revision)
     add_definitions(-DIDF_VER=\"${IDF_VER}\")
     git_submodule_check("${IDF_PATH}")
     set(IDF_VER ${IDF_VER} PARENT_SCOPE)
-endfunction()
+endfunction()

tools/cmake/utilities.cmake

@@ -73,30 +73,6 @@ function(move_if_different source destination)
 
 endfunction()
 
-
-# add_compile_options variant for C++ code only
-#
-# This adds global options, set target properties for
-# component-specific flags
-function(add_cxx_compile_options)
-    foreach(option ${ARGV})
-        # note: the Visual Studio Generator doesn't support this...
-        add_compile_options($<$<COMPILE_LANGUAGE:CXX>:${option}>)
-    endforeach()
-endfunction()
-
-# add_compile_options variant for C code only
-#
-# This adds global options, set target properties for
-# component-specific flags
-function(add_c_compile_options)
-    foreach(option ${ARGV})
-        # note: the Visual Studio Generator doesn't support this...
-        add_compile_options($<$<COMPILE_LANGUAGE:C>:${option}>)
-    endforeach()
-endfunction()
-
-
 # target_add_binary_data adds binary data into the built target,
 # by converting it to a generated source file which is then compiled
 # to a binary object as part of the build

tools/kconfig/Makefile

@@ -334,7 +334,7 @@ conf-idf: $(conf-objs)
 zconf.tab.c: zconf.lex.c
 
 zconf.lex.c: $(SRCDIR)/zconf.l
-	flex -L -P zconf -o zconf.lex.c $<
+	flex -L -Pzconf -ozconf.lex.c $<
 
 zconf.hash.c: $(SRCDIR)/zconf.gperf
 # strip CRs on Windows systems where gperf will otherwise barf on them

tools/kconfig_new/confgen.py

@@ -210,9 +210,9 @@ def write_json_menus(config, filename):
                 greatest_range = None
                 if len(sym.ranges) > 0:
                     # Note: Evaluating the condition using kconfiglib's expr_value
-                    # should have one result different from value 0 ("n").
+                    # should have one condition which is true
                     for min_range, max_range, cond_expr in sym.ranges:
-                        if kconfiglib.expr_value(cond_expr) != "n":
+                        if kconfiglib.expr_value(cond_expr):
                             greatest_range = [min_range, max_range]
                 new_json["range"] = greatest_range
 
@@ -221,9 +221,9 @@ def write_json_menus(config, filename):
             greatest_range = None
             if len(sym.ranges) > 0:
                 # Note: Evaluating the condition using kconfiglib's expr_value
-                # should have one result different from value 0 ("n").
+                # should have one condition which is true
                 for min_range, max_range, cond_expr in sym.ranges:
-                    if kconfiglib.expr_value(cond_expr) != "n":
+                    if kconfiglib.expr_value(cond_expr):
                         greatest_range = [int(min_range.str_value), int(max_range.str_value)]
 
             new_json = {

tools/unit-test-app/components/unity/ref_clock.c

@@ -119,6 +119,8 @@ void ref_clock_init()
     PCNT.ctrl.val |= BIT(REF_CLOCK_PCNT_UNIT * 2);
     PCNT.ctrl.val &= ~BIT(REF_CLOCK_PCNT_UNIT * 2);
 
+    ets_delay_us(10000);
+
     // Enable interrupt
     s_milliseconds = 0;
     ESP_ERROR_CHECK(esp_intr_alloc(ETS_PCNT_INTR_SOURCE, ESP_INTR_FLAG_IRAM, pcnt_isr, NULL, &s_intr_handle));