From 0a95151a75af0aa9bd76ce2f167fbb17f3eb3d8e Mon Sep 17 00:00:00 2001 From: Marius Vikhammer Date: Fri, 4 Dec 2020 11:35:21 +0800 Subject: [PATCH] unit_test: Refactor all performance tests that rely on cache compensated timer There is no ccomp timer on C3, which means our performance tests will start failing again due to variance caused by cache misses. This MR adds TEST_PERFORMANCE_CCOMP_ macro that will only fail performance test if CCOMP timer is supported on the target --- components/driver/test/test_sdio.c | 12 ++++++------ components/esp32/test/test_sha.c | 4 ++-- components/esp32s2/test/test_sha.c | 4 ++-- components/mbedtls/test/test_aes_perf.c | 2 +- components/mbedtls/test/test_rsa.c | 8 ++++---- components/mbedtls/test/test_sha_perf.c | 2 +- components/soc/esp32/include/soc/soc_caps.h | 1 + components/soc/esp32s2/include/soc/soc_caps.h | 1 + components/soc/esp32s3/include/soc/soc_caps.h | 2 ++ components/spi_flash/test/test_esp_flash.c | 4 ++-- components/spi_flash/test/test_spi_flash.c | 4 ++-- components/vfs/test/test_vfs_fd.c | 4 ++-- .../components/test_utils/include/test_utils.h | 16 ++++++++++++++++ 13 files changed, 42 insertions(+), 22 deletions(-) diff --git a/components/driver/test/test_sdio.c b/components/driver/test/test_sdio.c index bc929bce43..7d24942f64 100644 --- a/components/driver/test/test_sdio.c +++ b/components/driver/test/test_sdio.c @@ -355,13 +355,13 @@ static void log_performance_tohost(uint32_t speed, const sdio_test_config_t* con if (!config->check_data) { switch (config->sdio_mode) { case SDIO_4BIT: - TEST_PERFORMANCE_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_TOHOST_4BIT, "%d", speed); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_TOHOST_4BIT, "%d", speed); break; case SDIO_1BIT: - TEST_PERFORMANCE_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_TOHOST_1BIT, "%d", speed); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_TOHOST_1BIT, "%d", speed); break; case SDIO_SPI: - TEST_PERFORMANCE_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_TOHOST_SPI, "%d", speed); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_TOHOST_SPI, "%d", speed); break; } } @@ -441,13 +441,13 @@ static void log_performance_frhost(uint32_t speed, const sdio_test_config_t* con if (!config->check_data) { switch (config->sdio_mode) { case SDIO_4BIT: - TEST_PERFORMANCE_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_FRHOST_4BIT, "%d", speed); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_FRHOST_4BIT, "%d", speed); break; case SDIO_1BIT: - TEST_PERFORMANCE_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_FRHOST_1BIT, "%d", speed); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_FRHOST_1BIT, "%d", speed); break; case SDIO_SPI: - TEST_PERFORMANCE_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_FRHOST_SPI, "%d", speed); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_FRHOST_SPI, "%d", speed); break; } } diff --git a/components/esp32/test/test_sha.c b/components/esp32/test/test_sha.c index b2795171b0..075191475c 100644 --- a/components/esp32/test/test_sha.c +++ b/components/esp32/test/test_sha.c @@ -63,8 +63,8 @@ TEST_CASE("Test esp_sha()", "[hw_crypto]") free(buffer); - TEST_PERFORMANCE_LESS_THAN(TIME_SHA1_32KB, "%dus", us_sha1); - TEST_PERFORMANCE_LESS_THAN(TIME_SHA512_32KB, "%dus", us_sha512); + TEST_PERFORMANCE_CCOMP_LESS_THAN(TIME_SHA1_32KB, "%dus", us_sha1); + TEST_PERFORMANCE_CCOMP_LESS_THAN(TIME_SHA512_32KB, "%dus", us_sha512); } TEST_CASE("Test esp_sha() function with long input", "[hw_crypto]") diff --git a/components/esp32s2/test/test_sha.c b/components/esp32s2/test/test_sha.c index 42bac29f8d..6a57faf08b 100644 --- a/components/esp32s2/test/test_sha.c +++ b/components/esp32s2/test/test_sha.c @@ -58,8 +58,8 @@ TEST_CASE("Test esp_sha()", "[hw_crypto]") free(buffer); - TEST_PERFORMANCE_LESS_THAN(TIME_SHA1_32KB, "%dus", us_sha1); - TEST_PERFORMANCE_LESS_THAN(TIME_SHA512_32KB, "%dus", us_sha512); + TEST_PERFORMANCE_CCOMP_LESS_THAN(TIME_SHA1_32KB, "%dus", us_sha1); + TEST_PERFORMANCE_CCOMP_LESS_THAN(TIME_SHA512_32KB, "%dus", us_sha512); } TEST_CASE("Test esp_sha() function with long input", "[hw_crypto]") diff --git a/components/mbedtls/test/test_aes_perf.c b/components/mbedtls/test/test_aes_perf.c index 5a455f4f38..6224ed7b40 100644 --- a/components/mbedtls/test/test_aes_perf.c +++ b/components/mbedtls/test/test_aes_perf.c @@ -64,6 +64,6 @@ TEST_CASE("mbedtls AES performance", "[aes][timeout=60]") printf("Encryption rate %.3fMB/sec\n", mb_sec); #ifdef CONFIG_MBEDTLS_HARDWARE_AES // Don't put a hard limit on software AES performance - TEST_PERFORMANCE_GREATER_THAN(AES_CBC_THROUGHPUT_MBSEC, "%.3fMB/sec", mb_sec); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(AES_CBC_THROUGHPUT_MBSEC, "%.3fMB/sec", mb_sec); #endif } diff --git a/components/mbedtls/test/test_rsa.c b/components/mbedtls/test/test_rsa.c index 1ef1f07465..ed4fd4e738 100644 --- a/components/mbedtls/test/test_rsa.c +++ b/components/mbedtls/test/test_rsa.c @@ -366,11 +366,11 @@ static void rsa_key_operations(int keysize, bool check_performance, bool use_bli private_perf = ccomp_timer_stop(); if (check_performance && keysize == 2048) { - TEST_PERFORMANCE_LESS_THAN(RSA_2048KEY_PUBLIC_OP, "public operations %d us", public_perf); - TEST_PERFORMANCE_LESS_THAN(RSA_2048KEY_PRIVATE_OP, "private operations %d us", private_perf); + TEST_PERFORMANCE_CCOMP_LESS_THAN(RSA_2048KEY_PUBLIC_OP, "public operations %d us", public_perf); + TEST_PERFORMANCE_CCOMP_LESS_THAN(RSA_2048KEY_PRIVATE_OP, "private operations %d us", private_perf); } else if (check_performance && keysize == 4096) { - TEST_PERFORMANCE_LESS_THAN(RSA_4096KEY_PUBLIC_OP, "public operations %d us", public_perf); - TEST_PERFORMANCE_LESS_THAN(RSA_4096KEY_PRIVATE_OP, "private operations %d us", private_perf); + TEST_PERFORMANCE_CCOMP_LESS_THAN(RSA_4096KEY_PUBLIC_OP, "public operations %d us", public_perf); + TEST_PERFORMANCE_CCOMP_LESS_THAN(RSA_4096KEY_PRIVATE_OP, "private operations %d us", private_perf); } TEST_ASSERT_EQUAL_MEMORY_MESSAGE(orig_buf, decrypted_buf, keysize / 8, "RSA operation"); diff --git a/components/mbedtls/test/test_sha_perf.c b/components/mbedtls/test/test_sha_perf.c index c21085fb26..28e7a166ed 100644 --- a/components/mbedtls/test/test_sha_perf.c +++ b/components/mbedtls/test/test_sha_perf.c @@ -50,6 +50,6 @@ TEST_CASE("mbedtls SHA performance", "[aes]") printf("SHA256 rate %.3fMB/sec\n", mb_sec); #ifdef CONFIG_MBEDTLS_HARDWARE_SHA // Don't put a hard limit on software SHA performance - TEST_PERFORMANCE_GREATER_THAN(SHA256_THROUGHPUT_MBSEC, "%.3fMB/sec", mb_sec); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SHA256_THROUGHPUT_MBSEC, "%.3fMB/sec", mb_sec); #endif } diff --git a/components/soc/esp32/include/soc/soc_caps.h b/components/soc/esp32/include/soc/soc_caps.h index e5e2393e48..9f37a65d81 100644 --- a/components/soc/esp32/include/soc/soc_caps.h +++ b/components/soc/esp32/include/soc/soc_caps.h @@ -71,6 +71,7 @@ #define SOC_CPU_CORES_NUM 2 #define SOC_ULP_SUPPORTED 1 #define SOC_RTC_SLOW_MEM_SUPPORTED 1 +#define SOC_CCOMP_TIMER_SUPPORTED 1 /*-------------------------- ADC CAPS ----------------------------------------*/ #define SOC_ADC_PERIPH_NUM (2) diff --git a/components/soc/esp32s2/include/soc/soc_caps.h b/components/soc/esp32s2/include/soc/soc_caps.h index ff620ac6ad..0d1436ef52 100644 --- a/components/soc/esp32s2/include/soc/soc_caps.h +++ b/components/soc/esp32s2/include/soc/soc_caps.h @@ -47,6 +47,7 @@ #define SOC_PCNT_SUPPORTED 1 #define SOC_ULP_SUPPORTED 1 #define SOC_RTC_SLOW_MEM_SUPPORTED 1 +#define SOC_CCOMP_TIMER_SUPPORTED 1 #define SOC_CACHE_SUPPORT_WRAP 1 diff --git a/components/soc/esp32s3/include/soc/soc_caps.h b/components/soc/esp32s3/include/soc/soc_caps.h index d080d9e9e2..d3e493ac4a 100644 --- a/components/soc/esp32s3/include/soc/soc_caps.h +++ b/components/soc/esp32s3/include/soc/soc_caps.h @@ -14,6 +14,8 @@ #define SOC_CACHE_SUPPORT_WRAP 1 #define SOC_ULP_SUPPORTED 1 #define SOC_RTC_SLOW_MEM_SUPPORTED 1 +#define SOC_CCOMP_TIMER_SUPPORTED 1 + /*-------------------------- ADC CAPS ----------------------------------------*/ #include "adc_caps.h" diff --git a/components/spi_flash/test/test_esp_flash.c b/components/spi_flash/test/test_esp_flash.c index 7588fa1511..6722891d03 100644 --- a/components/spi_flash/test/test_esp_flash.c +++ b/components/spi_flash/test/test_esp_flash.c @@ -1008,8 +1008,8 @@ static void test_flash_read_write_performance(const esp_partition_t *part) TEST_ASSERT_EQUAL_HEX8_ARRAY(data_to_write, data_read, total_len); #if !CONFIG_SPIRAM && !CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE -# define CHECK_DATA(bus, suffix) TEST_PERFORMANCE_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_##bus##suffix, "%d", speed_##suffix) -# define CHECK_ERASE(bus, var) TEST_PERFORMANCE_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_##bus##ERASE, "%d", var) +# define CHECK_DATA(bus, suffix) TEST_PERFORMANCE_CCOMP_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_##bus##suffix, "%d", speed_##suffix) +# define CHECK_ERASE(bus, var) TEST_PERFORMANCE_CCOMP_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_##bus##ERASE, "%d", var) #else # define CHECK_DATA(bus, suffix) ((void)speed_##suffix) # define CHECK_ERASE(bus, var) ((void)var) diff --git a/components/spi_flash/test/test_spi_flash.c b/components/spi_flash/test/test_spi_flash.c index bb7673ca63..99cc7110a7 100644 --- a/components/spi_flash/test/test_spi_flash.c +++ b/components/spi_flash/test/test_spi_flash.c @@ -312,8 +312,8 @@ TEST_CASE("Test spi_flash read/write performance", "[spi_flash]") // Data checks are disabled when PSRAM is used or in Freertos compliance check test #if !CONFIG_SPIRAM && !CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE -# define CHECK_DATA(suffix) TEST_PERFORMANCE_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_LEGACY_##suffix, "%d", speed_##suffix) -# define CHECK_ERASE(var) TEST_PERFORMANCE_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_LEGACY_ERASE, "%d", var) +# define CHECK_DATA(suffix) TEST_PERFORMANCE_CCOMP_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_LEGACY_##suffix, "%d", speed_##suffix) +# define CHECK_ERASE(var) TEST_PERFORMANCE_CCOMP_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_LEGACY_ERASE, "%d", var) #else # define CHECK_DATA(suffix) ((void)speed_##suffix) # define CHECK_ERASE(var) ((void)var) diff --git a/components/vfs/test/test_vfs_fd.c b/components/vfs/test/test_vfs_fd.c index 639ec417b1..29a3e86821 100644 --- a/components/vfs/test/test_vfs_fd.c +++ b/components/vfs/test/test_vfs_fd.c @@ -260,9 +260,9 @@ TEST_CASE("Open & write & close through VFS passes performance test", "[vfs]") const int ns_per_iter = (int) (time_diff_us * 1000 / iter_count); TEST_ESP_OK( esp_vfs_unregister(VFS_PREF1) ); #ifdef CONFIG_SPIRAM - TEST_PERFORMANCE_LESS_THAN(VFS_OPEN_WRITE_CLOSE_TIME_PSRAM, "%dns", ns_per_iter); + TEST_PERFORMANCE_CCOMP_LESS_THAN(VFS_OPEN_WRITE_CLOSE_TIME_PSRAM, "%dns", ns_per_iter); #else - TEST_PERFORMANCE_LESS_THAN(VFS_OPEN_WRITE_CLOSE_TIME, "%dns", ns_per_iter); + TEST_PERFORMANCE_CCOMP_LESS_THAN(VFS_OPEN_WRITE_CLOSE_TIME, "%dns", ns_per_iter); #endif } diff --git a/tools/unit-test-app/components/test_utils/include/test_utils.h b/tools/unit-test-app/components/test_utils/include/test_utils.h index 2cfb2a5ea2..77281ac794 100644 --- a/tools/unit-test-app/components/test_utils/include/test_utils.h +++ b/tools/unit-test-app/components/test_utils/include/test_utils.h @@ -21,6 +21,7 @@ #include "freertos/FreeRTOS.h" #include "freertos/task.h" #include "unity.h" +#include "soc/soc_caps.h" /* include performance pass standards header file */ #include "idf_performance.h" #include "idf_performance_target.h" @@ -50,6 +51,21 @@ extern "C" { TEST_ASSERT(value > PERFORMANCE_CON(IDF_PERFORMANCE_MIN_, name)); \ } while(0) +/* Macros to be used when performance is calculated using the cache compensated timer + will not assert if ccomp not supported */ +#if SOC_CCOMP_TIMER_SUPPORTED +#define TEST_PERFORMANCE_CCOMP_GREATER_THAN(name, value_fmt, value) \ + TEST_PERFORMANCE_GREATER_THAN(name, value_fmt, value) +#define TEST_PERFORMANCE_CCOMP_LESS_THAN(name, value_fmt, value) \ + TEST_PERFORMANCE_LESS_THAN(name, value_fmt, value) +#else +#define TEST_PERFORMANCE_CCOMP_GREATER_THAN(name, value_fmt, value) \ + printf("[Performance][" PERFORMANCE_STR(name) "]: "value_fmt"\n", value); +#define TEST_PERFORMANCE_CCOMP_LESS_THAN(name, value_fmt, value) \ + printf("[Performance][" PERFORMANCE_STR(name) "]: "value_fmt"\n", value); +#endif //SOC_CCOMP_TIMER_SUPPORTED + + /* @brief macro to print IDF performance * @param mode : performance item name. a string pointer. * @param value_fmt: print format and unit of the value, for example: "%02fms", "%dKB"