diff --git a/components/esp_driver_isp/CMakeLists.txt b/components/esp_driver_isp/CMakeLists.txt index 7ce185791b..b3b04938f6 100644 --- a/components/esp_driver_isp/CMakeLists.txt +++ b/components/esp_driver_isp/CMakeLists.txt @@ -11,6 +11,7 @@ set(requires) if(CONFIG_SOC_ISP_SUPPORTED) list(APPEND srcs "src/isp_core.c" "src/isp_af.c" + "src/isp_ccm.c" "src/isp_awb.c") endif() diff --git a/components/esp_driver_isp/include/driver/isp.h b/components/esp_driver_isp/include/driver/isp.h index 38088eb938..7a36bb172c 100644 --- a/components/esp_driver_isp/include/driver/isp.h +++ b/components/esp_driver_isp/include/driver/isp.h @@ -15,3 +15,4 @@ #include "driver/isp_af.h" #include "driver/isp_awb.h" #include "driver/isp_bf.h" +#include "driver/isp_ccm.h" diff --git a/components/esp_driver_isp/include/driver/isp_awb.h b/components/esp_driver_isp/include/driver/isp_awb.h index 2846e9778b..ee85513e94 100644 --- a/components/esp_driver_isp/include/driver/isp_awb.h +++ b/components/esp_driver_isp/include/driver/isp_awb.h @@ -177,7 +177,7 @@ typedef struct { /** * @brief Prototype of ISP AWB event callback * - * @param[in] handle ISP AWB controller handle + * @param[in] awb_ctlr ISP AWB controller handle * @param[in] edata ISP AWB event data * @param[in] user_data User registered context, registered when in `esp_isp_awb_env_detector_register_event_callbacks()` * diff --git a/components/esp_driver_isp/include/driver/isp_ccm.h b/components/esp_driver_isp/include/driver/isp_ccm.h new file mode 100644 index 0000000000..451c69bcba --- /dev/null +++ b/components/esp_driver_isp/include/driver/isp_ccm.h @@ -0,0 +1,70 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include "esp_err.h" +#include "driver/isp_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Color Correction Matrix configurations + * + */ +typedef struct { + float matrix[ISP_CCM_DIMENSION][ISP_CCM_DIMENSION]; /*!< The color correction matrix in float, range (-4.0, 4.0) */ + bool saturation; /*!< Whether to use saturation when the float data in the matrix is out of the range, + * For example, if one of the matrix data is 5.0, + * When saturation is true, and final value will be limited to 4.0, and won't rise error + * When saturation is false, `esp_isp_ccm_configure` will rise ESP_ERR_INVALID_ARG error + */ +} esp_isp_ccm_config_t; + +/** + * @brief ISP Color Correction Matrix (CCM) configuration + * + * @note This function is allowed to be called before or after `esp_isp_ccm_enable`, + * but it only takes effect until `esp_isp_ccm_enable` is called + * + * @param[in] proc Processor handle + * @param[in] ccm_cfg CCM configurations, set NULL to de-configure the ISP CCM + * + * @return + * - ESP_OK On success + * - ESP_ERR_INVALID_ARG If the combination of arguments is invalid + */ +esp_err_t esp_isp_ccm_configure(isp_proc_handle_t proc, const esp_isp_ccm_config_t *ccm_cfg); + +/** + * @brief Enable ISP CCM function + * + * @param[in] proc Processor handle + * + * @return + * - ESP_OK On success + * - ESP_ERR_INVALID_ARG If the combination of arguments is invalid. + */ +esp_err_t esp_isp_ccm_enable(isp_proc_handle_t proc); + +/** + * @brief Disable ISP CCM function + * + * @param[in] proc Processor handle + * + * @return + * - ESP_OK On success + * - ESP_ERR_INVALID_ARG If the combination of arguments is invalid. + */ +esp_err_t esp_isp_ccm_disable(isp_proc_handle_t proc); + +#ifdef __cplusplus +} +#endif diff --git a/components/esp_driver_isp/src/isp_ccm.c b/components/esp_driver_isp/src/isp_ccm.c new file mode 100644 index 0000000000..5fb4598488 --- /dev/null +++ b/components/esp_driver_isp/src/isp_ccm.c @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include "esp_log.h" +#include "esp_check.h" +#include "freertos/FreeRTOS.h" +#include "driver/isp_core.h" +#include "driver/isp_ccm.h" +#include "esp_private/isp_private.h" + +static const char *TAG = "ISP_CCM"; + +/*--------------------------------------------------------------- + CCM +---------------------------------------------------------------*/ +esp_err_t esp_isp_ccm_configure(isp_proc_handle_t proc, const esp_isp_ccm_config_t *ccm_cfg) +{ + ESP_RETURN_ON_FALSE(proc && ccm_cfg, ESP_ERR_INVALID_ARG, TAG, "invalid argument: null pointer"); + + bool ret = true; + portENTER_CRITICAL(&proc->spinlock); + ret = isp_hal_ccm_set_matrix(&proc->hal, ccm_cfg->saturation, ccm_cfg->matrix); + portEXIT_CRITICAL(&proc->spinlock); + ESP_RETURN_ON_FALSE(ret, ESP_ERR_INVALID_ARG, TAG, "invalid argument: ccm matrix contain NaN or out of range"); + + return ESP_OK; +} + +esp_err_t esp_isp_ccm_enable(isp_proc_handle_t proc) +{ + ESP_RETURN_ON_FALSE(proc, ESP_ERR_INVALID_ARG, TAG, "invalid argument: null pointer"); + + portENTER_CRITICAL(&proc->spinlock); + isp_ll_ccm_clk_enable(proc->hal.hw, true); + isp_ll_ccm_enable(proc->hal.hw, true); + portEXIT_CRITICAL(&proc->spinlock); + + return ESP_OK; +} + +esp_err_t esp_isp_ccm_disable(isp_proc_handle_t proc) +{ + ESP_RETURN_ON_FALSE(proc, ESP_ERR_INVALID_ARG, TAG, "invalid argument: null pointer"); + + portENTER_CRITICAL(&proc->spinlock); + isp_ll_ccm_enable(proc->hal.hw, false); + isp_ll_ccm_clk_enable(proc->hal.hw, false); + portEXIT_CRITICAL(&proc->spinlock); + + return ESP_OK; +} diff --git a/components/esp_driver_isp/test_apps/isp/main/test_isp_driver.c b/components/esp_driver_isp/test_apps/isp/main/test_isp_driver.c index 3ed3467242..d2f660ea5d 100644 --- a/components/esp_driver_isp/test_apps/isp/main/test_isp_driver.c +++ b/components/esp_driver_isp/test_apps/isp/main/test_isp_driver.c @@ -118,3 +118,38 @@ TEST_CASE("ISP AWB driver basic function", "[isp]") TEST_ESP_OK(esp_isp_disable(isp_proc)); TEST_ESP_OK(esp_isp_del_processor(isp_proc)); } + +TEST_CASE("ISP CCM basic function", "[isp]") +{ + esp_isp_processor_cfg_t isp_config = { + .clk_hz = 80 * 1000 * 1000, + .input_data_source = ISP_INPUT_DATA_SOURCE_CSI, + .input_data_color_type = ISP_COLOR_RAW8, + .output_data_color_type = ISP_COLOR_RGB565, + }; + isp_proc_handle_t isp_proc = NULL; + TEST_ESP_OK(esp_isp_new_processor(&isp_config, &isp_proc)); + TEST_ESP_OK(esp_isp_enable(isp_proc)); + + esp_isp_ccm_config_t ccm_cfg = { + .matrix = { + {5.0, 0.0, 0.0}, + {0.0, 1.0, 0.0}, + {0.0, 0.0, 1.0} + }, + .saturation = false, + }; + // Out of range case + TEST_ESP_ERR(ESP_ERR_INVALID_ARG, esp_isp_ccm_configure(isp_proc, &ccm_cfg)); + // saturation case + ccm_cfg.saturation = true; + TEST_ESP_OK(esp_isp_ccm_configure(isp_proc, &ccm_cfg)); + TEST_ESP_OK(esp_isp_ccm_enable(isp_proc)); + // Allow to be called after enabled + ccm_cfg.matrix[0][0] = -1.1; + TEST_ESP_OK(esp_isp_ccm_configure(isp_proc, &ccm_cfg)); + TEST_ESP_OK(esp_isp_ccm_disable(isp_proc)); + + TEST_ESP_OK(esp_isp_disable(isp_proc)); + TEST_ESP_OK(esp_isp_del_processor(isp_proc)); +} diff --git a/components/hal/esp32p4/include/hal/isp_ll.h b/components/hal/esp32p4/include/hal/isp_ll.h index fbac2e908c..8792659fcb 100644 --- a/components/hal/esp32p4/include/hal/isp_ll.h +++ b/components/hal/esp32p4/include/hal/isp_ll.h @@ -100,6 +100,22 @@ typedef union { uint32_t val; } isp_ll_awb_rgb_ratio_t; +/*--------------------------------------------------------------- + CCM +---------------------------------------------------------------*/ +#define ISP_LL_CCM_MATRIX_INT_BITS (2) +#define ISP_LL_CCM_MATRIX_FRAC_BITS (10) +#define ISP_LL_CCM_MATRIX_TOT_BITS (ISP_LL_CCM_MATRIX_INT_BITS + ISP_LL_CCM_MATRIX_FRAC_BITS + 1) // including one sign bit + +typedef union { + struct { + uint32_t fraction: ISP_LL_AWB_RGB_RATIO_FRAC_BITS; + uint32_t integer: ISP_LL_AWB_RGB_RATIO_INT_BITS; + uint32_t sign: 1; + }; + uint32_t val; +} isp_ll_ccm_gain_t; + /** * @brief Env monitor mode */ @@ -790,6 +806,25 @@ static inline void isp_ll_ccm_enable(isp_dev_t *hw, bool enable) hw->cntl.ccm_en = enable; } +/** + * @brief Set the Color Correction Matrix + * + * @param[in] hw Hardware instance address + * @param[in] fixed_point_matrix Color Correction Matrix in fixed-point format + */ +static inline void isp_ll_ccm_set_matrix(isp_dev_t *hw, isp_ll_ccm_gain_t fixed_point_matrix[ISP_CCM_DIMENSION][ISP_CCM_DIMENSION]) +{ + hw->ccm_coef0.ccm_rr = fixed_point_matrix[0][0].val; + hw->ccm_coef0.ccm_rg = fixed_point_matrix[0][1].val; + hw->ccm_coef1.ccm_rb = fixed_point_matrix[0][2].val; + hw->ccm_coef1.ccm_gr = fixed_point_matrix[1][0].val; + hw->ccm_coef3.ccm_gg = fixed_point_matrix[1][1].val; + hw->ccm_coef3.ccm_gb = fixed_point_matrix[1][2].val; + hw->ccm_coef4.ccm_br = fixed_point_matrix[2][0].val; + hw->ccm_coef4.ccm_bg = fixed_point_matrix[2][1].val; + hw->ccm_coef5.ccm_bb = fixed_point_matrix[2][2].val; +} + /*--------------------------------------------------------------- Color ---------------------------------------------------------------*/ diff --git a/components/hal/include/hal/isp_hal.h b/components/hal/include/hal/isp_hal.h index fd291e7ec5..d8566a9ec4 100644 --- a/components/hal/include/hal/isp_hal.h +++ b/components/hal/include/hal/isp_hal.h @@ -88,6 +88,21 @@ uint32_t isp_hal_check_clear_intr_event(const isp_hal_context_t *hal, uint32_t m */ void isp_hal_bf_config(isp_hal_context_t *hal, isp_hal_bf_cfg_t *config); +/*--------------------------------------------------------------- + Color Correction Matrix +---------------------------------------------------------------*/ +/** + * @brief Set Color Correction Matrix + * + * @param[in] hal Context of the HAL layer + * @param[in] saturation Whether to enable saturation when float data overflow + * @param[in] flt_matrix 3x3 RGB correction matrix + * @return + * - true Set success + * - false Invalid argument + */ +bool isp_hal_ccm_set_matrix(const isp_hal_context_t *hal, bool saturation, const float flt_matrix[ISP_CCM_DIMENSION][ISP_CCM_DIMENSION]); + /*--------------------------------------------------------------- AWB ---------------------------------------------------------------*/ diff --git a/components/hal/include/hal/isp_types.h b/components/hal/include/hal/isp_types.h index a664d8001a..d36b8a09fc 100644 --- a/components/hal/include/hal/isp_types.h +++ b/components/hal/include/hal/isp_types.h @@ -100,6 +100,15 @@ typedef enum { ISP_BF_EDGE_PADDING_MODE_CUSTOM_DATA, ///< Fill BF edge padding data with custom pixel data } isp_bf_edge_padding_mode_t; +/*--------------------------------------------------------------- + CCM +---------------------------------------------------------------*/ +#if SOC_ISP_CCM_SUPPORTED +#define ISP_CCM_DIMENSION SOC_ISP_CCM_DIMENSION ///< ISP Color Correction Matrix dimension +#else +#define ISP_CCM_DIMENSION 0 ///< Not support CCM +#endif + /*--------------------------------------------------------------- AWB ---------------------------------------------------------------*/ diff --git a/components/hal/isp_hal.c b/components/hal/isp_hal.c index 7d33af5c5d..37e69fd772 100644 --- a/components/hal/isp_hal.c +++ b/components/hal/isp_hal.c @@ -70,6 +70,31 @@ uint32_t isp_hal_check_clear_intr_event(const isp_hal_context_t *hal, uint32_t m return triggered_events; } + +/*--------------------------------------------------------------- + Color Correction Matrix +---------------------------------------------------------------*/ +bool isp_hal_ccm_set_matrix(const isp_hal_context_t *hal, bool saturation, const float flt_matrix[ISP_CCM_DIMENSION][ISP_CCM_DIMENSION]) +{ + isp_ll_ccm_gain_t fp_matrix[ISP_CCM_DIMENSION][ISP_CCM_DIMENSION] = {}; + hal_utils_fixed_point_t fp_cfg = { + .int_bit = ISP_LL_CCM_MATRIX_INT_BITS, + .frac_bit = ISP_LL_CCM_MATRIX_FRAC_BITS, + .saturation = saturation, + }; + int err_level = saturation ? -1 : 0; + /* Transfer the float type to fixed point */ + for (int i = 0; i < ISP_CCM_DIMENSION; i++) { + for (int j = 0; j < ISP_CCM_DIMENSION; j++) { + if (hal_utils_float_to_fixed_point_32b(flt_matrix[i][j], &fp_cfg, &fp_matrix[i][j].val) < err_level) { + return false; + } + } + } + isp_ll_ccm_set_matrix(hal->hw, fp_matrix); + return true; +} + /*--------------------------------------------------------------- AWB ---------------------------------------------------------------*/ diff --git a/components/soc/esp32p4/include/soc/Kconfig.soc_caps.in b/components/soc/esp32p4/include/soc/Kconfig.soc_caps.in index bfe0ed7664..4e1d152e60 100644 --- a/components/soc/esp32p4/include/soc/Kconfig.soc_caps.in +++ b/components/soc/esp32p4/include/soc/Kconfig.soc_caps.in @@ -803,6 +803,10 @@ config SOC_ISP_BF_SUPPORTED bool default y +config SOC_ISP_CCM_SUPPORTED + bool + default y + config SOC_ISP_DVP_SUPPORTED bool default y @@ -835,6 +839,10 @@ config SOC_ISP_BF_TEMPLATE_Y_NUMS int default 3 +config SOC_ISP_CCM_DIMENSION + int + default 3 + config SOC_ISP_DVP_DATA_WIDTH_MAX int default 16 diff --git a/components/soc/esp32p4/include/soc/soc_caps.h b/components/soc/esp32p4/include/soc/soc_caps.h index 55f3b8565a..f774eaf9a8 100644 --- a/components/soc/esp32p4/include/soc/soc_caps.h +++ b/components/soc/esp32p4/include/soc/soc_caps.h @@ -321,6 +321,7 @@ /*-------------------------- ISP CAPS ----------------------------------------*/ #define SOC_ISP_BF_SUPPORTED 1 +#define SOC_ISP_CCM_SUPPORTED 1 #define SOC_ISP_DVP_SUPPORTED 1 #define SOC_ISP_NUMS 1U @@ -330,6 +331,7 @@ #define SOC_ISP_SHARE_CSI_BRG 1 #define SOC_ISP_BF_TEMPLATE_X_NUMS 3 #define SOC_ISP_BF_TEMPLATE_Y_NUMS 3 +#define SOC_ISP_CCM_DIMENSION 3 #define SOC_ISP_DVP_DATA_WIDTH_MAX 16 /*-------------------------- LEDC CAPS ---------------------------------------*/ diff --git a/docs/doxygen/Doxyfile_esp32p4 b/docs/doxygen/Doxyfile_esp32p4 index cd1724d538..01cc245bcd 100644 --- a/docs/doxygen/Doxyfile_esp32p4 +++ b/docs/doxygen/Doxyfile_esp32p4 @@ -27,6 +27,8 @@ INPUT += \ $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp.h \ $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp_types.h \ $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp_af.h \ + $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp_awb.h \ + $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp_ccm.h \ $(PROJECT_PATH)/components/esp_driver_jpeg/include/driver/jpeg_decode.h \ $(PROJECT_PATH)/components/esp_driver_jpeg/include/driver/jpeg_encode.h \ $(PROJECT_PATH)/components/esp_driver_ppa/include/driver/ppa.h \ diff --git a/docs/en/api-reference/peripherals/isp.rst b/docs/en/api-reference/peripherals/isp.rst index 15220418a2..ed4e725085 100644 --- a/docs/en/api-reference/peripherals/isp.rst +++ b/docs/en/api-reference/peripherals/isp.rst @@ -15,8 +15,10 @@ Terminology - RAW: Unprocessed data directly output from an image sensor, typically divided into R, Gr, Gb, and B four channels classified into RAW8, RAW10, RAW12, etc., based on bit width - RGB: Colored image format composed of red, green, and blue colors classified into RGB888, RGB565, etc., based on the bit width of each color - YUV: Colored image format composed of luminance and chrominance classified into YUV444, YUV422, YUV420, etc., based on the data arrangement + - BF: Bayer Domain Filter - AF: Auto-focus - AWB: Auto-white balance + - CCM: Color correction matrix ISP Pipeline ------------ @@ -59,6 +61,7 @@ The ISP driver offers following services: - `Enable and disable ISP processor <#isp-enable-disable>`__ - covers how to enable and disable an ISP processor. - `Get AF statistics in one shot or continuous way <#isp-af-statistics>`__ - covers how to get AF statistics one-shot or continuously. - `Get AWB statistics in one shot or continuous way <#isp-awb-statistics>`__ - covers how to get AWB white patches statistics one-shot or continuously. +- `Configure CCM <#isp-ccm-config>`__ - covers how to config the Color Correction Matrix. - `Register callback <#isp-callback>`__ - covers how to hook user specific code to ISP driver event callback function. - `Thread Safety <#isp-thread-safety>`__ - lists which APIs are guaranteed to be thread safe by the driver. - `Kconfig Options <#isp-kconfig-options>`__ - lists the supported Kconfig options that can bring different effects to the driver. @@ -304,6 +307,36 @@ Note that if you want to use the continuous statistics, you need to register the /* Delete the awb controller and free the resources */ ESP_ERROR_CHECK(esp_isp_del_awb_controller(awb_ctlr)); +.. _isp-ccm-config: + +Configure CCM +^^^^^^^^^^^^^ + +Color Correction Matrix can scale the color ratio of RGB888 pixels. It can be used for adjusting the image color via some algorithms, for example, used for white balance by inputting the AWB computed result, or used as a Filter with some filter algorithms. + +To adjust the color correction matrix, you can refer to the following code: + +.. code-block:: c + + // ... + // Configure CCM + esp_isp_ccm_config_t ccm_cfg = { + .matrix = { + 1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0 + }, + .saturation = false, + }; + ESP_ERROR_CHECK(esp_isp_ccm_configure(isp_proc, &ccm_cfg)); + // The configured CCM will be applied to the image once the CCM module is enabled + ESP_ERROR_CHECK(esp_isp_ccm_enable(isp_proc)); + // CCM can also be configured after it is enabled + ccm_cfg.matrix[0][0] = 2.0; + ESP_ERROR_CHECK(esp_isp_ccm_configure(isp_proc, &ccm_cfg)); + // Disable CCM if no longer needed + ESP_ERROR_CHECK(esp_isp_ccm_disable(isp_proc)); + .. _isp-callback: Register Event Callbacks