Dump the frame of the other core when interrupt watchdog happens

components/esp32/cpu_start.c

@@ -1,4 +1,4 @@
-// Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
+// Copyright 2015-2018 Espressif Systems (Shanghai) PTE LTD
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -328,6 +328,8 @@ void start_cpu0_default(void)
     do_global_ctors();
 #if CONFIG_INT_WDT
     esp_int_wdt_init();
+    //Initialize the interrupt watch dog for CPU0.
+    esp_int_wdt_cpu_init();
 #endif
     esp_cache_err_int_init();
     esp_crosscore_int_init();
@@ -377,6 +379,10 @@ void start_cpu1_default(void)
 #if CONFIG_ESP32_APPTRACE_ENABLE
     esp_err_t err = esp_apptrace_init();
     assert(err == ESP_OK && "Failed to init apptrace module on APP CPU!");
+#endif
+#if CONFIG_INT_WDT
+    //Initialize the interrupt watch dog for CPU1.
+    esp_int_wdt_cpu_init();
 #endif
     //Take care putting stuff here: if asked, FreeRTOS will happily tell you the scheduler
     //has started, but it isn't active *on this CPU* yet.

components/esp32/include/esp_int_wdt.h

@@ -1,4 +1,4 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+// Copyright 2015-2018 Espressif Systems (Shanghai) PTE LTD
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -29,7 +29,7 @@ interrupts for too long, or code within interrupt handlers taking too long.
 It does this by setting up a watchdog which gets fed from the FreeRTOS
 task switch interrupt. When this watchdog times out, initially it will call
 a high-level interrupt routine that will panic FreeRTOS in order to allow
-for forensic examination of the state of the CPU. When this interrupt 
+for forensic examination of the state of the both CPUs. When this interrupt
 handler is not called and the watchdog times out a second time, it will 
 reset the SoC.
 
@@ -38,12 +38,22 @@ This uses the TIMERG1 WDT.
 
 
 /**
-  * @brief  Initialize the interrupt watchdog. This is called in the init code if
+  * @brief  Initialize the non-CPU-specific parts of interrupt watchdog.
-  *         the interrupt watchdog is enabled in menuconfig.
+  *         This is called in the init code if the interrupt watchdog
+  *         is enabled in menuconfig.
   *
   */
 void esp_int_wdt_init();
 
+/**
+  * @brief  Enable the interrupt watchdog on the current CPU. This is called
+  *         in the init code by both CPUs if the interrupt watchdog is enabled
+  *         in menuconfig.
+  *
+  */
+void esp_int_wdt_cpu_init();
+
+
 
 /**
   * @}

components/esp32/int_wdt.c

@@ -1,4 +1,4 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+// Copyright 2015-2018 Espressif Systems (Shanghai) PTE LTD
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -88,10 +88,11 @@ void esp_int_wdt_init() {
     TIMERG1.wdt_wprotect=0;
     TIMERG1.int_clr_timers.wdt=1;
     timer_group_intr_enable(TIMER_GROUP_1, TIMG_WDT_INT_ENA_M);
-    esp_register_freertos_tick_hook_for_cpu(tick_hook, 0);
+}
-#ifndef CONFIG_FREERTOS_UNICORE
+
-    esp_register_freertos_tick_hook_for_cpu(tick_hook, 1);
+void esp_int_wdt_cpu_init()
-#endif
+{
+    esp_register_freertos_tick_hook_for_cpu(tick_hook, xPortGetCoreID());
     ESP_INTR_DISABLE(WDT_INT_NUM);
     intr_matrix_set(xPortGetCoreID(), ETS_TG1_WDT_LEVEL_INTR_SOURCE, WDT_INT_NUM);
     //We do not register a handler for the interrupt because it is interrupt level 4 which

components/esp32/panic.c

@@ -186,6 +186,12 @@ static void setFirstBreakpoint(uint32_t pc)
         ::"r"(pc):"a3", "a4");
 }
 
+//When interrupt watchdog happen in one core, both cores will be interrupted.
+//The core which doesn't trigger the interrupt watchdog will save the frame and return.
+//The core which triggers the interrupt watchdog will use the saved frame, and dump frames for both cores.
+#if !CONFIG_FREERTOS_UNICORE
+static volatile XtExcFrame * other_core_frame = NULL;
+#endif //!CONFIG_FREERTOS_UNICORE
 
 void panicHandler(XtExcFrame *frame)
 {
@@ -206,11 +212,26 @@ void panicHandler(XtExcFrame *frame)
     if (frame->exccause <= PANIC_RSN_MAX) {
         reason = reasons[frame->exccause];
     }
+
+#if !CONFIG_FREERTOS_UNICORE
+    //Save frame for other core.
+    if ((frame->exccause == PANIC_RSN_INTWDT_CPU0 && core_id == 1) || (frame->exccause == PANIC_RSN_INTWDT_CPU1 && core_id == 0)) {
+        other_core_frame = frame;
+        while (1);
+    }
+
+    //The core which triggers the interrupt watchdog will delay 1 us, so the other core can save its frame.
+    if (frame->exccause == PANIC_RSN_INTWDT_CPU0 || frame->exccause == PANIC_RSN_INTWDT_CPU1) {
+        ets_delay_us(1);
+    }
+
     if (frame->exccause == PANIC_RSN_CACHEERR && esp_cache_err_get_cpuid() != core_id) {
         // Cache error interrupt will be handled by the panic handler
         // on the other CPU.
-        return;
+        while (1);
     }
+#endif //!CONFIG_FREERTOS_UNICORE
+
     haltOtherCore();
     esp_dport_access_int_abort();
     panicPutStr("Guru Meditation Error: Core ");
@@ -428,10 +449,9 @@ static void doBacktrace(XtExcFrame *frame)
 }
 
 /*
-  We arrive here after a panic or unhandled exception, when no OCD is detected. Dump the registers to the
+ * Dump registers and do backtrace.
-  serial port and either jump to the gdb stub, halt the CPU or reboot.
+ */
-*/
+static void commonErrorHandler_dump(XtExcFrame *frame, int core_id)
-static __attribute__((noreturn)) void commonErrorHandler(XtExcFrame *frame)
 {
     int *regs = (int *)frame;
     int x, y;
@@ -441,17 +461,13 @@ static __attribute__((noreturn)) void commonErrorHandler(XtExcFrame *frame)
         "A14     ", "A15     ", "SAR     ", "EXCCAUSE", "EXCVADDR", "LBEG    ", "LEND    ", "LCOUNT  "
     };
 
-    // start panic WDT to restart system if we hang in this handler
-    esp_panic_wdt_start();
-
-    //Feed the watchdogs, so they will give us time to print out debug info
-    reconfigureAllWdts();
-
     /* only dump registers for 'real' crashes, if crashing via abort()
        the register window is no longer useful.
     */
     if (!abort_called) {
-        panicPutStr("Register dump:\r\n");
+        panicPutStr("Core");
+        panicPutDec(core_id);
+        panicPutStr(" register dump:\r\n");
 
         for (x = 0; x < 24; x += 4) {
             for (y = 0; y < 4; y++) {
@@ -464,11 +480,65 @@ static __attribute__((noreturn)) void commonErrorHandler(XtExcFrame *frame)
             }
             panicPutStr("\r\n");
         }
+
+        if (xPortInterruptedFromISRContext()
+#if !CONFIG_FREERTOS_UNICORE
+            && other_core_frame != frame
+#endif //!CONFIG_FREERTOS_UNICORE
+            ) {
+            //If the core which triggers the interrupt watchdog was in ISR context, dump the epc registers.
+            uint32_t __value;
+            panicPutStr("Core");
+            panicPutDec(core_id);
+            panicPutStr(" was running in ISR context:\r\n");
+
+            __asm__("rsr.epc1 %0" : "=a"(__value));
+            panicPutStr("EPC1    : 0x");
+            panicPutHex(__value);
+
+            __asm__("rsr.epc2 %0" : "=a"(__value));
+            panicPutStr("  EPC2    : 0x");
+            panicPutHex(__value);
+
+            __asm__("rsr.epc3 %0" : "=a"(__value));
+            panicPutStr("  EPC3    : 0x");
+            panicPutHex(__value);
+
+            __asm__("rsr.epc4 %0" : "=a"(__value));
+            panicPutStr("  EPC4    : 0x");
+            panicPutHex(__value);
+
+            panicPutStr("\r\n");
+        }
+
     }
 
     /* With windowed ABI backtracing is easy, let's do it. */
     doBacktrace(frame);
 
+}
+
+/*
+  We arrive here after a panic or unhandled exception, when no OCD is detected. Dump the registers to the
+  serial port and either jump to the gdb stub, halt the CPU or reboot.
+*/
+static __attribute__((noreturn)) void commonErrorHandler(XtExcFrame *frame)
+{
+
+    int core_id = xPortGetCoreID();
+    // start panic WDT to restart system if we hang in this handler
+    esp_panic_wdt_start();
+
+    //Feed the watchdogs, so they will give us time to print out debug info
+    reconfigureAllWdts();
+
+    commonErrorHandler_dump(frame, core_id);
+#if !CONFIG_FREERTOS_UNICORE
+    if (other_core_frame != NULL) {
+        commonErrorHandler_dump((XtExcFrame *)other_core_frame, (core_id ? 0 : 1));
+    }
+#endif //!CONFIG_FREERTOS_UNICORE
+
 #if CONFIG_ESP32_APPTRACE_ENABLE
     disableAllWdts();
 #if CONFIG_SYSVIEW_ENABLE

components/freertos/include/freertos/portable.h

@@ -183,6 +183,12 @@ void vPortSetStackWatchpoint( void* pxStackStart );
  */
 BaseType_t xPortInIsrContext();
 
+/*
+ * This function will be called in High prio ISRs. Returns true if the current core was in ISR context
+ * before calling into high prio ISR context.
+ */
+BaseType_t xPortInterruptedFromISRContext();
+
 /*
  * The structures and methods of manipulating the MPU are contained within the
  * port layer.

components/freertos/port.c

@@ -288,6 +288,14 @@ BaseType_t xPortInIsrContext()
 	return ret;
 }
 
+/*
+ * This function will be called in High prio ISRs. Returns true if the current core was in ISR context
+ * before calling into high prio ISR context.
+ */
+BaseType_t IRAM_ATTR xPortInterruptedFromISRContext()
+{
+	return (port_interruptNesting[xPortGetCoreID()] != 0);
+}
 
 void vPortAssertIfInISR()
 {