diff --git a/components/freertos/port/xtensa/xtensa_vectors.S b/components/freertos/port/xtensa/xtensa_vectors.S
index 1560ff27d6..f38c71269f 100644
--- a/components/freertos/port/xtensa/xtensa_vectors.S
+++ b/components/freertos/port/xtensa/xtensa_vectors.S
@@ -903,34 +903,32 @@ _xt_coproc_exc:
     /* Get co-processor state save area of new owner thread. */
     call0   XT_RTOS_CP_STATE                /* a15 = new owner's save area */
 
-    #ifndef CONFIG_FREERTOS_FPU_IN_ISR
-    beqz    a15, .L_goto_invalid
+    #if CONFIG_FREERTOS_FPU_IN_ISR
+    beqz    a15, .L_skip_core_pin           /* CP used in ISR, skip task pinning */
+    #else
+    beqz    a15, .L_goto_invalid            /* not in a thread (invalid) */
     #endif
 
-    /*When FPU in ISR is enabled we could deal with zeroed a15 */
+#if configNUM_CORES > 1
+    /* CP operations are incompatible with unpinned tasks. Thus we pin the task
+    to the current running core. */
+    movi    a2, pxCurrentTCB
+    getcoreid a3                            /* a3 = current core ID */
+    addx4   a2, a3, a2
+    l32i    a2, a2, 0                       /* a2 = start of pxCurrentTCB[cpuid] */
+    addi    a2, a2, TASKTCB_XCOREID_OFFSET  /* a2 = &TCB.xCoreID */
+    s32i    a3, a2, 0                       /* TCB.xCoreID = current core ID */
+#endif // configNUM_CORES > 1
+
+#if CONFIG_FREERTOS_FPU_IN_ISR
+.L_skip_core_pin:
+#endif
 
     /* Enable the co-processor's bit in CPENABLE. */
     movi    a0, _xt_coproc_mask
     rsr     a4, CPENABLE                    /* a4 = CPENABLE */
     addx4   a0, a5, a0                      /* a0 = &_xt_coproc_mask[n] */
     l32i    a0, a0, 0                       /* a0 = (n << 16) | (1 << n) */
-
-    /* FPU operations are incompatible with non-pinned tasks. If we have a FPU operation
-       here, to keep the entire thing from crashing, it's better to pin the task to whatever
-       core we're running on now. */
-    movi    a2, pxCurrentTCB
-    getcoreid a3
-    addx4     a2,  a3, a2
-    l32i    a2, a2, 0                       /* a2 = start of pxCurrentTCB[cpuid] */
-    addi    a2, a2, TASKTCB_XCOREID_OFFSET  /* offset to xCoreID in tcb struct */
-    s32i    a3, a2, 0                       /* store current cpuid */
-
-    /* Grab correct xt_coproc_owner_sa for this core */
-    movi    a2, XCHAL_CP_MAX << 2
-    mull    a2, a2, a3                      /* multiply by current processor id */
-    movi    a3, _xt_coproc_owner_sa         /* a3 = base of owner array */
-    add     a3, a3, a2                      /* a3 = owner area needed for this processor */
-
     extui   a2, a0, 0, 16                   /* coprocessor bitmask portion */
     or      a4, a4, a2                      /* a4 = CPENABLE | (1 << n) */
     wsr     a4, CPENABLE
@@ -941,6 +939,11 @@ everywhere): _xt_coproc_release assumes it works like this in order not to need
 locking.
 */
 
+    /* Grab correct xt_coproc_owner_sa for this core */
+    movi    a2, XCHAL_CP_MAX << 2
+    mull    a2, a2, a3                      /* multiply by current processor id */
+    movi    a3, _xt_coproc_owner_sa         /* a3 = base of owner array */
+    add     a3, a3, a2                      /* a3 = owner area needed for this processor */
 
     /* Get old coprocessor owner thread (save area ptr) and assign new one.  */
     addx4   a3,  a5, a3                      /* a3 = &_xt_coproc_owner_sa[n] */