Added support for Perl style \N \P and \p.

Completed first draft of Unicode UCS-4 support.
Broken compiler compatibility fixes.
Added unicode_iterators.


[SVN r26185]

include/boost/regex/config.hpp

@@ -179,7 +179,7 @@
 #  define BOOST_REGEX_DECL
 #endif
 
-#if (defined(BOOST_MSVC) || defined(__BORLANDC__)) && !defined(BOOST_REGEX_NO_LIB) && !defined(BOOST_REGEX_SOURCE) && !defined(BOOST_ALL_NO_LIB) && defined(__cplusplus)
+#if !defined(BOOST_REGEX_NO_LIB) && !defined(BOOST_REGEX_SOURCE) && !defined(BOOST_ALL_NO_LIB) && defined(__cplusplus)
 #  define BOOST_LIB_NAME boost_regex
 #  if defined(BOOST_REGEX_DYN_LINK) || defined(BOOST_ALL_DYN_LINK)
 #     define BOOST_DYN_LINK

include/boost/regex/icu.hpp

@@ -177,6 +177,8 @@ private:
       offset_xdigit = U_CHAR_CATEGORY_COUNT+2,
       offset_underscore = U_CHAR_CATEGORY_COUNT+3,
       offset_unicode = U_CHAR_CATEGORY_COUNT+4,
+      offset_any = U_CHAR_CATEGORY_COUNT+5,
+      offset_ascii = U_CHAR_CATEGORY_COUNT+6,
    };
 
    //
@@ -187,6 +189,10 @@ private:
    static const char_class_type mask_xdigit;
    static const char_class_type mask_underscore;
    static const char_class_type mask_unicode;
+   static const char_class_type mask_any;
+   static const char_class_type mask_ascii;
+
+   static char_class_type lookup_icu_mask(const ::UChar32* p1, const ::UChar32* p2);
 
    boost::shared_ptr< ::boost::re_detail::icu_regex_traits_implementation> m_pimpl;
 };

include/boost/regex/pending/unicode_iterator.hpp

@@ -49,35 +49,63 @@ inline bool is_surrogate(T v)
    return (v & 0xF800u) == 0xd800;
 }
 
+inline unsigned utf8_byte_count(boost::uint8_t c)
+{
+   // if the most significant bit with a zero in it is in position
+   // 8-N then there are N bytes in this UTF-8 sequence:
+   boost::uint8_t mask = 0x80u;
+   unsigned result = 0;
+   while(c & mask)
+   {
+      ++result;
+      mask >>= 1;
+   }
+   return (result == 0) ? 1 : result;
+}
+
+inline unsigned utf8_trailing_byte_count(boost::uint8_t c)
+{
+   return utf8_byte_count(c) - 1;
+}
+
 }
 
 template <class BaseIterator, class U16Type = ::boost::uint16_t>
 class u32_to_u16_iterator
    : public boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type>
 {
-   typedef boost::iterator_facade<u32_to_u16_iterator<BaseIterator>, U16Type, std::bidirectional_iterator_tag, const U16Type> base_type;
+   typedef boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type> base_type;
+
+#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
    typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
 
    BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
    BOOST_STATIC_ASSERT(sizeof(U16Type)*CHAR_BIT == 16);
+#endif
+
 public:
    typename base_type::reference
       dereference()const
    {
       if(m_current == 2)
-         const_cast<u32_to_u16_iterator*>(this)->extract_current();
+         extract_current();
       return m_values[m_current];
    }
    bool equal(const u32_to_u16_iterator& that)const
    {
       if(m_position == that.m_position)
       {
+         // Both m_currents must be equal, or both even
+         // this is the same as saying their sum must be even:
+         return (m_current + that.m_current) & 1u ? false : true;
+            /*
          if((m_current >= 2) && (that.m_current < 2))
             const_cast<u32_to_u16_iterator*>(this)->extract_current();
          else if((m_current < 2) && (that.m_current >= 2))
             const_cast<u32_to_u16_iterator&>(that).extract_current();
          if(m_current == that.m_current)
             return true;
+            */
       }
       return false;
    }
@@ -127,7 +155,7 @@ public:
       m_values[2] = 0;
    }
 private:
-   void invalid_code_point(::boost::uint32_t val)
+   static void invalid_code_point(::boost::uint32_t val)
    {
 #ifndef BOOST_NO_STD_LOCALE
       std::stringstream ss;
@@ -139,15 +167,15 @@ private:
       boost::throw_exception(e);
    }
 
-   void extract_current()
+   void extract_current()const
    {
       // begin by checking for a code point out of range:
-      if(static_cast< ::boost::uint32_t>(*m_position) >= 0x10000u)
+      ::boost::uint32_t v = *m_position;
+      if(v >= 0x10000u)
       {
-         if(static_cast< ::boost::uint32_t>(*m_position) > 0x10FFFFu)
+         if(v > 0x10FFFFu)
             invalid_code_point(*m_position);
          // split into two surrogates:
-         base_value_type v = *m_position;
          m_values[0] = static_cast<U16Type>(v >> 10) + detail::high_surrogate_base;
          m_values[1] = static_cast<U16Type>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
          m_current = 0;
@@ -166,29 +194,31 @@ private:
       }
    }
    BaseIterator m_position;
-   U16Type m_values[3];
-   unsigned m_current;
+   mutable U16Type m_values[3];
+   mutable unsigned m_current;
 };
 
 template <class BaseIterator, class U32Type = ::boost::uint32_t>
 class u16_to_u32_iterator
    : public boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
 {
-   typedef boost::iterator_facade<u16_to_u32_iterator<BaseIterator>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
+   typedef boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
    // special values for pending iterator reads:
    BOOST_STATIC_CONSTANT(::boost::uint32_t, pending_read = 0xffffffffu);
 
+#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
    typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
 
    BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 16);
    BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
+#endif
 
 public:
    typename base_type::reference
       dereference()const
    {
       if(m_value == pending_read)
-         const_cast<u16_to_u32_iterator*>(this)->extract_current();
+         extract_current();
       return m_value;
    }
    bool equal(const u16_to_u32_iterator& that)const
@@ -223,7 +253,7 @@ public:
       m_value = pending_read;
    }
 private:
-   void invalid_code_point(::boost::uint16_t val)
+   static void invalid_code_point(::boost::uint16_t val)
    {
 #ifndef BOOST_NO_STD_LOCALE
       std::stringstream ss;
@@ -234,28 +264,254 @@ private:
 #endif
       boost::throw_exception(e);
    }
-   void extract_current()
+   void extract_current()const
    {
       m_value = static_cast<U32Type>(static_cast< ::boost::uint16_t>(*m_position));
       // if the last value is a high surrogate then adjust m_position and m_value as needed:
       if(detail::is_high_surrogate(*m_position))
       {
          // precondition; next value must have be a low-surrogate:
-         ::boost::uint16_t t = *++m_position;
-         if((*m_position & 0xFC00u) != 0xDC00u)
+         BaseIterator next(m_position);
+         ::boost::uint16_t t = *++next;
+         if((t & 0xFC00u) != 0xDC00u)
             invalid_code_point(t);
          m_value = (m_value - detail::high_surrogate_base) << 10;
          m_value |= (static_cast<U32Type>(static_cast< ::boost::uint16_t>(t)) & detail::ten_bit_mask);
-         --m_position;
       }
       // postcondition; result must not be a surrogate:
       if(detail::is_surrogate(m_value))
          invalid_code_point(static_cast< ::boost::uint16_t>(m_value));
    }
    BaseIterator m_position;
-   U32Type m_value;
+   mutable U32Type m_value;
 };
 
-}
+template <class BaseIterator, class U8Type = ::boost::uint8_t>
+class u32_to_u8_iterator
+   : public boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type>
+{
+   typedef boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type> base_type;
+   
+#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
+   typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
+
+   BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
+   BOOST_STATIC_ASSERT(sizeof(U8Type)*CHAR_BIT == 8);
+#endif
+
+public:
+   typename base_type::reference
+      dereference()const
+   {
+      if(m_current == 4)
+         extract_current();
+      return m_values[m_current];
+   }
+   bool equal(const u32_to_u8_iterator& that)const
+   {
+      if(m_position == that.m_position)
+      {
+         // either the m_current's must be equal, or one must be 0 and 
+         // the other 4: which means neither must have bits 1 or 2 set:
+         return (m_current == that.m_current)
+            || (((m_current | that.m_current) & 3) == 0);
+      }
+      return false;
+   }
+   void increment()
+   {
+      // if we have a pending read then read now, so that we know whether
+      // to skip a position, or move to a low-surrogate:
+      if(m_current == 4)
+      {
+         // pending read:
+         extract_current();
+      }
+      // move to the next surrogate position:
+      ++m_current;
+      // if we've reached the end skip a position:
+      if(m_values[m_current] == 0)
+      {
+         m_current = 4;
+         ++m_position;
+      }
+   }
+   void decrement()
+   {
+      if((m_current & 3) == 0)
+      {
+         --m_position;
+         extract_current();
+         m_current = 3;
+         while(m_current && (m_values[m_current] == 0))
+            --m_current;
+      }
+      else
+         --m_current;
+   }
+   BaseIterator base()const
+   {
+      return m_position;
+   }
+   // construct:
+   u32_to_u8_iterator() : m_position(), m_current(0)
+   {
+      m_values[4] = 0;
+   }
+   u32_to_u8_iterator(BaseIterator b) : m_position(b), m_current(4)
+   {
+      m_values[4] = 0;
+   }
+private:
+   static void invalid_code_point(::boost::uint32_t val)
+   {
+#ifndef BOOST_NO_STD_LOCALE
+      std::stringstream ss;
+      ss << "Invalid UTF-32 code point U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-8 sequence";
+      std::out_of_range e(ss.str());
+#else
+      std::out_of_range e("Invalid UTF-32 code point encountered while trying to encode UTF-8 sequence");
+#endif
+      boost::throw_exception(e);
+   }
+
+   void extract_current()const
+   {
+      boost::uint32_t c = *m_position;
+      if(c > 0x10FFFFu)
+         invalid_code_point(c);
+      if(c < 0x80u)
+      {
+         m_values[0] = static_cast<unsigned char>(c);
+         m_values[1] = static_cast<unsigned char>(0u);
+         m_values[2] = static_cast<unsigned char>(0u);
+         m_values[3] = static_cast<unsigned char>(0u);
+      }
+      else if(c < 0x800u)
+      {
+         m_values[0] = static_cast<unsigned char>(0xC0u + (c >> 6));
+         m_values[1] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+         m_values[2] = static_cast<unsigned char>(0u);
+         m_values[3] = static_cast<unsigned char>(0u);
+      }
+      else if(c < 0x10000u)
+      {
+         m_values[0] = static_cast<unsigned char>(0xE0u + (c >> 12));
+         m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
+         m_values[2] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+         m_values[3] = static_cast<unsigned char>(0u);
+      }
+      else
+      {
+         m_values[0] = static_cast<unsigned char>(0xF0u + (c >> 18));
+         m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
+         m_values[2] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
+         m_values[3] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
+      }
+      m_current= 0;
+   }
+   BaseIterator m_position;
+   mutable U8Type m_values[5];
+   mutable unsigned m_current;
+};
+
+template <class BaseIterator, class U32Type = ::boost::uint32_t>
+class u8_to_u32_iterator
+   : public boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
+{
+   typedef boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
+   // special values for pending iterator reads:
+   BOOST_STATIC_CONSTANT(::boost::uint32_t, pending_read = 0xffffffffu);
+
+#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION)
+   typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
+
+   BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 8);
+   BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
+#endif
+
+public:
+   typename base_type::reference
+      dereference()const
+   {
+      if(m_value == pending_read)
+         extract_current();
+      return m_value;
+   }
+   bool equal(const u8_to_u32_iterator& that)const
+   {
+      return m_position == that.m_position;
+   }
+   void increment()
+   {
+      // skip high surrogate first if there is one:
+      unsigned c = detail::utf8_byte_count(*m_position);
+      std::advance(m_position, c);
+      m_value = pending_read;
+   }
+   void decrement()
+   {
+      // Keep backtracking until we don't have a trailing character:
+      unsigned count = 0;
+      while((*--m_position & 0xC0u) == 0x80u) ++count;
+      // now check that the sequence was valid:
+      if(count != detail::utf8_trailing_byte_count(*m_position))
+         invalid_sequnce();
+      m_value = pending_read;
+   }
+   BaseIterator base()const
+   {
+      return m_position;
+   }
+   // construct:
+   u8_to_u32_iterator() : m_position()
+   {
+      m_value = pending_read;
+   }
+   u8_to_u32_iterator(BaseIterator b) : m_position(b)
+   {
+      m_value = pending_read;
+   }
+private:
+   static void invalid_sequnce()
+   {
+      std::out_of_range e("Invalid UTF-8 sequence encountered while trying to encode UTF-32 character");
+      boost::throw_exception(e);
+   }
+   void extract_current()const
+   {
+      m_value = static_cast<U32Type>(static_cast< ::boost::uint8_t>(*m_position));
+      // we must not have a continuation character:
+      if((m_value & 0xC0u) == 0x80u)
+         invalid_sequnce();
+      // see how many extra byts we have:
+      unsigned extra = detail::utf8_trailing_byte_count(*m_position);
+      // extract the extra bits, 6 from each extra byte:
+      BaseIterator next(m_position);
+      for(unsigned c = 0; c < extra; ++c)
+      {
+         ++next;
+         m_value <<= 6;
+         m_value += static_cast<boost::uint8_t>(*next) & 0x3Fu;
+      }
+      // we now need to remove a few of the leftmost bits, but how many depends
+      // upon how many extra bytes we've extracted:
+      static const boost::uint32_t masks[] = 
+      {
+         0x7Fu,
+         0x7FFu,
+         0xFFFFu,
+         0x1FFFFFu,
+      };
+      m_value &= masks[extra];
+      // check the result:
+      if(m_value > 0x10FFFFu)
+         invalid_sequnce();
+   }
+   BaseIterator m_position;
+   mutable U32Type m_value;
+};
+
+} // namespace boost
 
 #endif // BOOST_REGEX_UNICODE_ITERATOR_HPP

include/boost/regex/v4/basic_regex_parser.hpp

@@ -611,6 +611,52 @@ bool basic_regex_parser<charT, traits>::parse_extended_escape()
       ++m_position;
       this->append_state(syntax_element_restart_continue);
       break;
+   case regex_constants::escape_type_not_property:
+      negate = true;
+      // fall through:
+   case regex_constants::escape_type_property:
+      {
+         ++m_position;
+         char_class_type m;
+         if(m_position == m_end)
+         {
+            fail(regex_constants::error_escape, m_position - m_base);
+            return false;
+         }
+         // maybe have \p{ddd}
+         if(this->m_traits.syntax_type(*m_position) == regex_constants::syntax_open_brace)
+         {
+            const charT* base = m_position;
+            // skip forward until we find enclosing brace:
+            while((m_position != m_end) && (this->m_traits.syntax_type(*m_position) != regex_constants::syntax_close_brace))
+               ++m_position;
+            if(m_position == m_end)
+            {
+               fail(regex_constants::error_escape, m_position - m_base);
+               return false;
+            }
+            m = this->m_traits.lookup_classname(++base, m_position++);
+         }
+         else
+         {
+            m = this->m_traits.lookup_classname(m_position, m_position+1);
+            ++m_position;
+         }
+         if(m != 0)
+         {
+            basic_char_set<charT, traits> char_set;
+            if(negate)
+               char_set.negate();
+            char_set.add_class(m);
+            if(0 == this->append_set(char_set))
+            {
+               fail(regex_constants::error_ctype, m_position - m_base);
+               return false;
+            }
+            return true;
+         }
+         fail(regex_constants::error_ctype, m_position - m_base);
+      }
    default:
       this->append_literal(unescape_character());
       break;
@@ -948,6 +994,7 @@ bool basic_regex_parser<charT, traits>::parse_set()
                if(m != 0)
                {
                   char_set.add_class(m);
+                  ++m_position;
                   break;
                }
             }
@@ -1373,6 +1420,41 @@ charT basic_regex_parser<charT, traits>::unescape_character()
       }
       return static_cast<charT>(val);
       }
+   case regex_constants::escape_type_named_char:
+      {
+         ++m_position;
+         if(m_position == m_end)
+         {
+            fail(regex_constants::error_escape, m_position - m_base);
+            return false;
+         }
+         // maybe have \N{name}
+         if(this->m_traits.syntax_type(*m_position) == regex_constants::syntax_open_brace)
+         {
+            const charT* base = m_position;
+            // skip forward until we find enclosing brace:
+            while((m_position != m_end) && (this->m_traits.syntax_type(*m_position) != regex_constants::syntax_close_brace))
+               ++m_position;
+            if(m_position == m_end)
+            {
+               fail(regex_constants::error_escape, m_position - m_base);
+               return false;
+            }
+            string_type s = this->m_traits.lookup_collatename(++base, m_position++);
+            if(s.empty())
+            {
+               fail(regex_constants::error_collate, m_position - m_base);
+               return false;
+            }
+            if(s.size() == 1)
+            {
+               return s[0];
+            }
+         }
+         // fall through is a failure:
+         fail(regex_constants::error_escape, m_position - m_base);
+         return false;
+      }
    default:
       result = *m_position;
       break;

include/boost/regex/v4/regex_format.hpp

@@ -75,7 +75,7 @@ class basic_regex_formatter
 public:
    typedef typename traits::char_type char_type;
    basic_regex_formatter(OutputIterator o, const Results& r, const traits& t)
-      : m_traits(t), m_results(r), m_out(o), m_state(output_copy) {}
+      : m_traits(t), m_results(r), m_out(o), m_state(output_copy), m_have_conditional(false) {}
    OutputIterator format(const char_type* p1, const char_type* p2, match_flag_type f);
    OutputIterator format(const char_type* p1, match_flag_type f)
    {
@@ -108,6 +108,7 @@ private:
    const char_type* m_end;       // format string end
    match_flag_type m_flags;      // format flags to use
    output_state    m_state;      // what to do with the next character
+   bool            m_have_conditional; // we are parsing a conditional
 private:
    basic_regex_formatter(const basic_regex_formatter&);
    basic_regex_formatter& operator=(const basic_regex_formatter&);
@@ -147,7 +148,10 @@ void basic_regex_formatter<OutputIterator, Results, traits>::format_all()
          if(m_flags & boost::regex_constants::format_all)
          {
             ++m_position;
+            bool have_conditional = m_have_conditional;
+            m_have_conditional = false;
             format_until_scope_end();
+            m_have_conditional = have_conditional;
             if(m_position == m_end)
                return;
             BOOST_ASSERT(*m_position == static_cast<char_type>(')'));
@@ -158,7 +162,6 @@ void basic_regex_formatter<OutputIterator, Results, traits>::format_all()
          ++m_position;
          break;
       case ')':
-      case ':':
          if(m_flags & boost::regex_constants::format_all)
          {
             return;
@@ -166,6 +169,14 @@ void basic_regex_formatter<OutputIterator, Results, traits>::format_all()
          put(*m_position);
          ++m_position;
          break;
+      case ':':
+         if((m_flags & boost::regex_constants::format_all) && m_have_conditional)
+         {
+            return;
+         }
+         put(*m_position);
+         ++m_position;
+         break;
       case '?':
          if(m_flags & boost::regex_constants::format_all)
          {
@@ -405,7 +416,9 @@ void basic_regex_formatter<OutputIterator, Results, traits>::format_conditional(
    // output varies depending upon whether sub-expression v matched or not:
    if(m_results[v].matched)
    {
+      m_have_conditional = true;
       format_all();
+      m_have_conditional = false;
       if((m_position != m_end) && (*m_position == static_cast<char_type>(':')))
       {
          // skip the ':':
@@ -425,7 +438,9 @@ void basic_regex_formatter<OutputIterator, Results, traits>::format_conditional(
       output_state saved_state = m_state;
       m_state = output_none;
       // format until ':' or ')':
+      m_have_conditional = true;
       format_all();
+      m_have_conditional = false;
       // restore state:
       m_state = saved_state;
       if((m_position != m_end) && (*m_position == static_cast<char_type>(':')))

include/boost/regex/v4/syntax_type.hpp

@@ -89,7 +89,11 @@ static const escape_syntax_type escape_type_C = 50;                            /
 static const escape_syntax_type escape_type_Z = 51;                            // for \Z
 static const escape_syntax_type escape_type_G = 52;                            // for \G
 
-static const escape_syntax_type syntax_max = 54;
+static const escape_syntax_type escape_type_property = 54;                     // for \p
+static const escape_syntax_type escape_type_not_property = 55;                 // for \P
+static const escape_syntax_type escape_type_named_char = 56;                   // for \N
+
+static const escape_syntax_type syntax_max = 57;
 
 }
 }