Merge branch 'feat/isp_bypass' into 'master'

isp: supported bypass isp to use csi, supported byte endianness swapping

Closes IDF-9706 and IDF-12620

See merge request espressif/esp-idf!37672

components/esp_driver_isp/include/driver/isp_core.h

@@ -33,6 +33,10 @@ typedef struct {
     uint32_t v_res;                               ///< Input vertical resolution, i.e. the number of lines in a frame
     color_raw_element_order_t bayer_order;        ///< Bayer order
     int intr_priority;                            ///< The interrupt priority, range 0~3, if set to 0, the driver will try to allocate an interrupt with a relative low priority (1,2,3)
+    struct {
+        uint32_t bypass_isp   : 1;                ///< Bypass ISP pipelines
+        uint32_t byte_swap_en : 1;                ///< Enable byte swap
+    } flags;                                      ///< Flags
 } esp_isp_processor_cfg_t;
 
 /**

components/esp_driver_isp/include/esp_private/isp_private.h

@@ -65,6 +65,7 @@ typedef struct isp_processor_t {
     uint32_t                    h_res;
     uint32_t                    v_res;
     color_raw_element_order_t   bayer_order;
+    bool                        bypass_isp;
     /* sub module contexts */
     isp_af_ctlr_t               af_ctlr[SOC_ISP_AF_CTLR_NUMS];
     isp_awb_ctlr_t              awb_ctlr;

components/esp_driver_isp/src/isp_core.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -17,12 +17,15 @@
 #include "esp_private/periph_ctrl.h"
 #include "esp_private/mipi_csi_share_hw_ctrl.h"
 #include "hal/hal_utils.h"
+#include "hal/color_hal.h"
 #include "soc/mipi_csi_bridge_struct.h"
 #include "soc/isp_periph.h"
 #include "soc/soc_caps.h"
 #include "esp_private/esp_clk_tree_common.h"
 #include "esp_private/isp_private.h"
 
+#define ISP_DIV_ROUND_UP(x, y)    (((x) + (y) - 1) / (y))
+
 typedef struct isp_platform_t {
     _lock_t         mutex;
     isp_processor_t *processors[SOC_ISP_NUMS];
@@ -76,6 +79,9 @@ esp_err_t esp_isp_new_processor(const esp_isp_processor_cfg_t *proc_config, isp_
     esp_err_t ret = ESP_FAIL;
     ESP_RETURN_ON_FALSE(proc_config && ret_proc, ESP_ERR_INVALID_ARG, TAG, "invalid argument: null pointer");
     ESP_RETURN_ON_FALSE(proc_config->input_data_source != ISP_INPUT_DATA_SOURCE_DWGDMA, ESP_ERR_NOT_SUPPORTED, TAG, "input source not supported yet");
+    if (proc_config->flags.bypass_isp) {
+        ESP_RETURN_ON_FALSE(proc_config->input_data_color_type == proc_config->output_data_color_type, ESP_ERR_INVALID_ARG, TAG, "isp is bypassed, input and output data color type should be same");
+    }
 
     isp_processor_t *proc = heap_caps_calloc(1, sizeof(isp_processor_t), ISP_MEM_ALLOC_CAPS);
     ESP_RETURN_ON_FALSE(proc, ESP_ERR_NO_MEM, TAG, "no mem");
@@ -117,27 +123,39 @@ esp_err_t esp_isp_new_processor(const esp_isp_processor_cfg_t *proc_config, isp_
     proc->isp_fsm = ISP_FSM_INIT;
     proc->spinlock = (portMUX_TYPE)portMUX_INITIALIZER_UNLOCKED;
 
-    //Input color format
-    bool valid_format = false;
+    //Input & Output color format
     color_space_pixel_format_t in_color_format = {
         .color_type_id = proc_config->input_data_color_type,
     };
-    valid_format = isp_ll_set_input_data_color_format(proc->hal.hw, in_color_format);
-    ESP_GOTO_ON_FALSE(valid_format, ESP_ERR_INVALID_ARG, err, TAG, "invalid input color space config");
-
-    //Output color format
-    valid_format = false;
     color_space_pixel_format_t out_color_format = {
         .color_type_id = proc_config->output_data_color_type,
     };
-    valid_format = isp_ll_set_output_data_color_format(proc->hal.hw, out_color_format);
-    ESP_GOTO_ON_FALSE(valid_format, ESP_ERR_INVALID_ARG, err, TAG, "invalid output color space config");
+    int in_bits_per_pixel = color_hal_pixel_format_get_bit_depth(in_color_format);
+
+    if (!proc_config->flags.bypass_isp) {
+        bool valid_format = false;
+        valid_format = isp_ll_set_input_data_color_format(proc->hal.hw, in_color_format);
+        ESP_GOTO_ON_FALSE(valid_format, ESP_ERR_INVALID_ARG, err, TAG, "invalid input color space config");
+
+        valid_format = false;
+        valid_format = isp_ll_set_output_data_color_format(proc->hal.hw, out_color_format);
+        ESP_GOTO_ON_FALSE(valid_format, ESP_ERR_INVALID_ARG, err, TAG, "invalid output color space config");
+    }
 
     isp_ll_clk_enable(proc->hal.hw, true);
     isp_ll_set_input_data_source(proc->hal.hw, proc_config->input_data_source);
     isp_ll_enable_line_start_packet_exist(proc->hal.hw, proc_config->has_line_start_packet);
     isp_ll_enable_line_end_packet_exist(proc->hal.hw, proc_config->has_line_end_packet);
-    isp_ll_set_intput_data_h_pixel_num(proc->hal.hw, proc_config->h_res);
+    if (proc_config->flags.bypass_isp) {
+        /**
+         * When ISP bypass, input module is still working, input Hsize needs to be re-calculated
+         * according to input bits per pixel and IDI32.
+         * Hsize now stands for the number of 32-bit in one line.
+         */
+        isp_ll_set_intput_data_h_pixel_num(proc->hal.hw, ISP_DIV_ROUND_UP(proc_config->h_res * in_bits_per_pixel, 32));
+    } else {
+        isp_ll_set_intput_data_h_pixel_num(proc->hal.hw, proc_config->h_res);
+    }
     isp_ll_set_intput_data_v_row_num(proc->hal.hw, proc_config->v_res);
     isp_ll_set_bayer_mode(proc->hal.hw, proc_config->bayer_order);
     isp_ll_yuv_set_std(proc->hal.hw, proc_config->yuv_std);
@@ -148,12 +166,16 @@ esp_err_t esp_isp_new_processor(const esp_isp_processor_cfg_t *proc_config, isp_
     if (out_color_format.color_space == COLOR_SPACE_RGB && proc_config->input_data_source == ISP_INPUT_DATA_SOURCE_DVP) {
         isp_ll_color_enable(proc->hal.hw, true); // workaround for DIG-474
     }
+    if (proc_config->flags.byte_swap_en) {
+        isp_ll_set_byte_swap(proc->hal.hw, true);
+    }
 
     proc->in_color_format = in_color_format;
     proc->out_color_format = out_color_format;
     proc->h_res = proc_config->h_res;
     proc->v_res = proc_config->v_res;
     proc->bayer_order = proc_config->bayer_order;
+    proc->bypass_isp = proc_config->flags.bypass_isp;
 
     *ret_proc = proc;
 
@@ -209,6 +231,7 @@ esp_err_t esp_isp_enable(isp_proc_handle_t proc)
 {
     ESP_RETURN_ON_FALSE(proc, ESP_ERR_INVALID_ARG, TAG, "invalid argument: null pointer");
     ESP_RETURN_ON_FALSE(proc->isp_fsm == ISP_FSM_INIT, ESP_ERR_INVALID_STATE, TAG, "processor isn't in init state");
+    ESP_RETURN_ON_FALSE(proc->bypass_isp == false, ESP_ERR_INVALID_STATE, TAG, "processor is configured to be bypassed");
 
     isp_ll_enable(proc->hal.hw, true);
     proc->isp_fsm = ISP_FSM_ENABLE;

components/hal/esp32p4/include/hal/isp_ll.h

@@ -499,6 +499,17 @@ static inline void isp_ll_set_bayer_mode(isp_dev_t *hw, color_raw_element_order_
     hw->frame_cfg.bayer_mode = bayer_order;
 }
 
+/**
+ * @brief Swap the data endianness order in bytes
+ *
+ * @param[in] hw           Hardware instance address
+ * @param[in] byte_swap_en byte swap enable or not
+ */
+static inline void isp_ll_set_byte_swap(isp_dev_t *hw, bool byte_swap_en)
+{
+    hw->cntl.byte_endian_order = byte_swap_en;
+}
+
 /*---------------------------------------------------------------
                       AF
 ---------------------------------------------------------------*/

docs/en/api-reference/peripherals/isp.rst

@@ -104,6 +104,11 @@ If the configurations in :cpp:type:`esp_isp_processor_cfg_t` is specified, users
 
 You can use the created handle to enable/disable the ISP driver and do other ISP module installation.
 
+.. note::
+
+    ISP peripheral is necessary if MIPI CSI or ISP_DVP is used as camera controller. This means even if ISP functions are not needed, you still need to install the ISP driver by calling :cpp:func:`esp_isp_new_processor`.
+
+    If ISP functions are not needed, ISP driver supports bypassing ISP pipelines and only necessary functions will be enabled. This can be done by setting :cpp:member:`esp_isp_processor_cfg_t::bypass_isp`.
 
 Install ISP Auto Focus (AF) Driver
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

examples/peripherals/camera/mipi_isp_dsi/README.md

@@ -143,6 +143,12 @@ I (1315) sensor_init: fmt[2].name:MIPI_2lane_24Minput_RAW8_800x800_50fps
 I (1355) sensor_init: Format in use:MIPI_2lane_24Minput_RAW8_800x640_50fps
 ```
 
+Below picture is from the video stream of OV5647 and ILI9881C. The camera module is not focused and has not been calibrated by the ESP on-chip ISP hardware.
+
+This image is also used as a reference, you can check output image after ISP auto-focus and other pipelines in [Example ISP Multi-pipeline](../../isp/multi_pipelines/).
+
+![CSI-ISP](image/csi2.jpg)
+
 
 ## Reference
 

examples/peripherals/isp/multi_pipelines/README.md

@@ -152,7 +152,12 @@ I (1315) cam_dsi: fmt[2].name:MIPI_2lane_24Minput_RAW8_800x800_50fps
 I (1355) cam_dsi: Format in use:MIPI_2lane_24Minput_RAW8_800x640_50fps
 ```
 
-You will also see the screen auto-focus when the screen image changes.
+Below picture is from the video stream of OV5647 and ILI9881C. The camera module is auto-focused and calibrated by ESP on-chip ISP hardware. The edge is over-sharpened as example code configured.
+
+This image is also used as a reference, you can check output image without ISP auto-focus and other pipelines in [Example MIPI Camera Display](../../camera/mipi_isp_dsi/).
+
+![ISP-MULTI-PIPELINE](image/isp2.jpg)
+
 
 ## Reference