diff --git a/CMakeLists.txt b/CMakeLists.txt
index 34e043dd41..6f09900ec4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,7 +20,9 @@ if(NOT BOOTLOADER_BUILD)
 
     if(CONFIG_COMPILER_OPTIMIZATION_SIZE)
         list(APPEND compile_options "-Os")
-        list(APPEND compile_options "-freorder-blocks")
+        if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+            list(APPEND compile_options "-freorder-blocks")
+        endif()
     elseif(CONFIG_COMPILER_OPTIMIZATION_DEFAULT)
         list(APPEND compile_options "-Og")
     elseif(CONFIG_COMPILER_OPTIMIZATION_NONE)
@@ -33,7 +35,9 @@ else()  # BOOTLOADER_BUILD
 
     if(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_SIZE)
         list(APPEND compile_options "-Os")
-        list(APPEND compile_options "-freorder-blocks")
+        if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+            list(APPEND compile_options "-freorder-blocks")
+        endif()
     elseif(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_DEBUG)
         list(APPEND compile_options "-Og")
     elseif(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_NONE)
@@ -74,6 +78,10 @@ if(CONFIG_COMPILER_DISABLE_GCC8_WARNINGS)
                                 "-Wno-int-in-bool-context")
 endif()
 
+if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+    list(APPEND c_compile_options "-Wno-old-style-declaration")
+endif()
+
 if(CONFIG_COMPILER_WARN_WRITE_STRINGS)
     list(APPEND compile_options "-Wwrite-strings")
 endif()
@@ -118,7 +126,15 @@ list(APPEND link_options "-fno-lto")
 # Placing jump tables in flash would cause issues with code that required
 # to be placed in IRAM
 list(APPEND compile_options "-fno-jump-tables")
-list(APPEND compile_options "-fno-tree-switch-conversion")
+if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+    # This flag is GCC-specific.
+    # Not clear yet if some other flag should be used for Clang.
+    list(APPEND compile_options "-fno-tree-switch-conversion")
+endif()
+
+if(CMAKE_C_COMPILER_ID MATCHES "LLVM")
+    list(APPEND compile_options "-fno-use-cxa-atexit")
+endif()
 
 idf_build_set_property(COMPILE_OPTIONS "${compile_options}" APPEND)
 idf_build_set_property(C_COMPILE_OPTIONS "${c_compile_options}" APPEND)
diff --git a/tools/cmake/build.cmake b/tools/cmake/build.cmake
index f67aaf991b..d0abf84386 100644
--- a/tools/cmake/build.cmake
+++ b/tools/cmake/build.cmake
@@ -107,8 +107,7 @@ function(__build_set_default_build_specifications)
                                     # go into the final binary so have no impact on size
                                     "-ggdb")
 
-    list(APPEND c_compile_options   "-std=gnu99"
-                                    "-Wno-old-style-declaration")
+    list(APPEND c_compile_options   "-std=gnu99")
 
     list(APPEND cxx_compile_options "-std=gnu++11")