diff --git a/configure.ac b/configure.ac
index 3ea9160e9..c4cd80c6a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -728,11 +728,6 @@ then
     AM_CFLAGS="$AM_CFLAGS -DNO_DEV_RANDOM -DNO_WRITEV -DNO_STDIO_FILESYSTEM -DWOLFSSL_NO_SOCK -DWOLFSSL_USER_IO"
 fi
 
-if test "$ENABLED_RNG" = "yes" || test "$ENABLED_BENCHMARK" = "yes"
-then
-    AC_SEARCH_LIBS([clock_gettime],[rt])
-fi
-
 
 # MATH LIBRARY SELECTION
 
@@ -6411,6 +6406,11 @@ then
     done
 fi
 
+if (test "$ENABLED_RNG" = "yes" && test "$ENABLED_ENTROPY_MEMUSE" != "no") || test "$ENABLED_BENCHMARK" = "yes"
+then
+    AC_SEARCH_LIBS([clock_gettime],[rt])
+fi
+
 
 # Filesystem Build
 if test "$ENABLED_LINUXKM" = "yes"