diff --git a/include/fmt/base.h b/include/fmt/base.h
index 29d0bc4c..b4b0a68d 100644
--- a/include/fmt/base.h
+++ b/include/fmt/base.h
@@ -213,6 +213,20 @@
 #  define FMT_DEPRECATED /* deprecated */
 #endif
 
+#ifndef FMT_NO_UNIQUE_ADDRESS
+#  if FMT_CPLUSPLUS >= 202002L
+#    if FMT_HAS_CPP_ATTRIBUTE(no_unique_address)
+#      define FMT_NO_UNIQUE_ADDRESS [[no_unique_address]]
+// VS2019 v16.10 and later except clang-cl (https://reviews.llvm.org/D110485).
+#    elif (FMT_MSC_VERSION >= 1929) && !FMT_CLANG_VERSION
+#      define FMT_NO_UNIQUE_ADDRESS [[msvc::no_unique_address]]
+#    endif
+#  endif
+#endif
+#ifndef FMT_NO_UNIQUE_ADDRESS
+#  define FMT_NO_UNIQUE_ADDRESS
+#endif
+
 #ifdef FMT_INLINE
 // Use the provided definition.
 #elif FMT_GCC_VERSION || FMT_CLANG_VERSION
@@ -1643,8 +1657,17 @@ struct is_output_iterator<
     void_t<decltype(*std::declval<decay_t<It>&>()++ = std::declval<T>())>>
     : std::true_type {};
 
+#ifdef FMT_USE_LOCALE
+// Use the provided definition.
+#elif defined(FMT_STATIC_THOUSANDS_SEPARATOR)
+#  define FMT_USE_LOCALE 0
+#else
+#  define FMT_USE_LOCALE 1
+#endif
+
 // A type-erased reference to an std::locale to avoid a heavy <locale> include.
-class locale_ref {
+struct locale_ref {
+#if FMT_USE_LOCALE
  private:
   const void* locale_;  // A type-erased pointer to std::locale.
 
@@ -1653,6 +1676,7 @@ class locale_ref {
   template <typename Locale> explicit locale_ref(const Locale& loc);
 
   explicit operator bool() const noexcept { return locale_ != nullptr; }
+#endif
 
   template <typename Locale> auto get() const -> Locale;
 };
@@ -2012,7 +2036,7 @@ class context {
  private:
   appender out_;
   basic_format_args<context> args_;
-  detail::locale_ref loc_;
+  FMT_NO_UNIQUE_ADDRESS detail::locale_ref loc_;
 
  public:
   /// The character type for the output.
diff --git a/include/fmt/format-inl.h b/include/fmt/format-inl.h
index 95c3e50b..8b883625 100644
--- a/include/fmt/format-inl.h
+++ b/include/fmt/format-inl.h
@@ -83,6 +83,11 @@ inline void fwrite_fully(const void* ptr, size_t count, FILE* stream) {
 using std::locale;
 using std::numpunct;
 using std::use_facet;
+
+template <typename Locale>
+locale_ref::locale_ref(const Locale& loc) : locale_(&loc) {
+  static_assert(std::is_same<Locale, locale>::value, "");
+}
 #else
 struct locale {};
 template <typename Char> struct numpunct {
@@ -93,14 +98,12 @@ template <typename Char> struct numpunct {
 template <typename Facet> Facet use_facet(locale) { return {}; }
 #endif  // FMT_USE_LOCALE
 
-template <typename Locale>
-locale_ref::locale_ref(const Locale& loc) : locale_(&loc) {
-  static_assert(std::is_same<Locale, locale>::value, "");
-}
-
 template <typename Locale> auto locale_ref::get() const -> Locale {
   static_assert(std::is_same<Locale, locale>::value, "");
-  return locale_ ? *static_cast<const locale*>(locale_) : locale();
+#if FMT_USE_LOCALE
+  if (locale_) return *static_cast<const locale*>(locale_);
+#endif
+  return locale();
 }
 
 template <typename Char>
@@ -1020,8 +1023,7 @@ template <> struct cache_accessor<double> {
       {0xe4d5e82392a40515, 0x0fabaf3feaa5334b},
       {0xb8da1662e7b00a17, 0x3d6a751f3b936244},
       {0x95527a5202df0ccb, 0x0f37801e0c43ebc9},
-      { 0xf13e34aabb430a15,
-        0x647726b9e7c68ff0 }
+      {0xf13e34aabb430a15, 0x647726b9e7c68ff0}
 #endif
     };
 
diff --git a/include/fmt/format.h b/include/fmt/format.h
index 85b2abd1..f7bc5195 100644
--- a/include/fmt/format.h
+++ b/include/fmt/format.h
@@ -73,20 +73,6 @@
 #  define FMT_INLINE_VARIABLE
 #endif
 
-#ifndef FMT_NO_UNIQUE_ADDRESS
-#  if FMT_CPLUSPLUS >= 202002L
-#    if FMT_HAS_CPP_ATTRIBUTE(no_unique_address)
-#      define FMT_NO_UNIQUE_ADDRESS [[no_unique_address]]
-// VS2019 v16.10 and later except clang-cl (https://reviews.llvm.org/D110485).
-#    elif (FMT_MSC_VERSION >= 1929) && !FMT_CLANG_VERSION
-#      define FMT_NO_UNIQUE_ADDRESS [[msvc::no_unique_address]]
-#    endif
-#  endif
-#endif
-#ifndef FMT_NO_UNIQUE_ADDRESS
-#  define FMT_NO_UNIQUE_ADDRESS
-#endif
-
 // Visibility when compiled as a shared library/object.
 #if defined(FMT_LIB_EXPORT) || defined(FMT_SHARED)
 #  define FMT_SO_VISIBILITY(value) FMT_VISIBILITY(value)
@@ -799,14 +785,6 @@ using is_double_double = bool_constant<std::numeric_limits<T>::digits == 106>;
 #  define FMT_USE_FULL_CACHE_DRAGONBOX 0
 #endif
 
-#ifdef FMT_USE_LOCALE
-// Use the provided definition.
-#elif defined(FMT_STATIC_THOUSANDS_SEPARATOR)
-#  define FMT_USE_LOCALE 0
-#else
-#  define FMT_USE_LOCALE 1
-#endif
-
 template <typename T, typename Enable = void>
 struct is_locale : std::false_type {};
 template <typename T>
@@ -2207,8 +2185,7 @@ template <typename Char = char> struct loc_writer {
 
 template <typename Char, typename OutputIt, typename T>
 FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, write_int_arg<T> arg,
-                                        const format_specs& specs, locale_ref)
-    -> OutputIt {
+                                        const format_specs& specs) -> OutputIt {
   static_assert(std::is_same<T, uint32_or_64_or_128_t<T>>::value, "");
   auto abs_value = arg.abs_value;
   auto prefix = arg.prefix;
@@ -2260,9 +2237,9 @@ FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, write_int_arg<T> arg,
 template <typename Char, typename OutputIt, typename T>
 FMT_CONSTEXPR FMT_NOINLINE auto write_int_noinline(OutputIt out,
                                                    write_int_arg<T> arg,
-                                                   const format_specs& specs,
-                                                   locale_ref loc) -> OutputIt {
-  return write_int<Char>(out, arg, specs, loc);
+                                                   const format_specs& specs)
+    -> OutputIt {
+  return write_int<Char>(out, arg, specs);
 }
 template <typename Char, typename T,
           FMT_ENABLE_IF(is_integral<T>::value &&
@@ -2273,7 +2250,7 @@ FMT_CONSTEXPR FMT_INLINE auto write(basic_appender<Char> out, T value,
     -> basic_appender<Char> {
   if (specs.localized() && write_loc(out, value, specs, loc)) return out;
   return write_int_noinline<Char>(out, make_write_int_arg(value, specs.sign()),
-                                  specs, loc);
+                                  specs);
 }
 // An inlined version of write used in format string compilation.
 template <typename Char, typename OutputIt, typename T,
@@ -2285,8 +2262,7 @@ FMT_CONSTEXPR FMT_INLINE auto write(OutputIt out, T value,
                                     const format_specs& specs, locale_ref loc)
     -> OutputIt {
   if (specs.localized() && write_loc(out, value, specs, loc)) return out;
-  return write_int<Char>(out, make_write_int_arg(value, specs.sign()), specs,
-                         loc);
+  return write_int<Char>(out, make_write_int_arg(value, specs.sign()), specs);
 }
 
 template <typename Char, typename OutputIt>