diff --git a/CHANGELOG.md b/CHANGELOG.md index 20d84a78..aa1a42f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +Version 183: + +* Fix a rare case of failed UTF8 validation + +-------------------------------------------------------------------------------- + Version 182: * Silence ubsan false positive diff --git a/doc/qbk/09_releases.qbk b/doc/qbk/09_releases.qbk index 3d392b00..fb6d6158 100644 --- a/doc/qbk/09_releases.qbk +++ b/doc/qbk/09_releases.qbk @@ -9,6 +9,14 @@ [section Release Notes] +[heading Boost 1.69] + +[*Fixes] + +* ([issue 1245]) Fix a rare case of incorrect UTF8 validation + +[heading Boost 1.68] + This version fixes a missing executor work guard in all composed operations used in the implementation. Users who are experiencing crashes related to asynchronous completion handlers are encouraged to upgrade. Also included @@ -16,8 +24,6 @@ is an improved mechanism for generating random numbers used to mask outgoing websocket frames when operating in the client mode. This resolves a vulnerability described in the Beast Hybrid Assessment Report from Bishop Fox. -[heading Boost 1.68] - [*New Features] The include directory `` contains features which are not diff --git a/include/boost/beast/websocket/detail/utf8_checker.hpp b/include/boost/beast/websocket/detail/utf8_checker.hpp index 63687587..ea6e61a9 100644 --- a/include/boost/beast/websocket/detail/utf8_checker.hpp +++ b/include/boost/beast/websocket/detail/utf8_checker.hpp @@ -112,7 +112,7 @@ write(std::uint8_t const* in, std::size_t size) if((p[0] & 0xe0) == 0xc0) { if( (p[1] & 0xc0) != 0x80 || - (p[0] & 0xfe) == 0xc0) // overlong + (p[0] & 0x1e) == 0) // overlong return false; p += 2; return true; @@ -121,8 +121,8 @@ write(std::uint8_t const* in, std::size_t size) { if( (p[1] & 0xc0) != 0x80 || (p[2] & 0xc0) != 0x80 - || (p[0] == 0xe0 && (p[1] & 0xe0) == 0x80) // overlong - || (p[0] == 0xed && (p[1] & 0xe0) == 0xa0) // surrogate + || (p[0] == 0xe0 && (p[1] & 0x20) == 0) // overlong + || (p[0] == 0xed && (p[1] & 0x20) == 0x20) // surrogate //|| (p[0] == 0xef && p[1] == 0xbf && (p[2] & 0xfe) == 0xbe) // U+FFFE or U+FFFF ) return false; @@ -131,10 +131,11 @@ write(std::uint8_t const* in, std::size_t size) } if((p[0] & 0xf8) == 0xf0) { - if( (p[1] & 0xc0) != 0x80 + if( (p[0] & 0x07) >= 0x05 // invalid F5...FF characters + || (p[1] & 0xc0) != 0x80 || (p[2] & 0xc0) != 0x80 || (p[3] & 0xc0) != 0x80 - || (p[0] == 0xf0 && (p[1] & 0xf0) == 0x80) // overlong + || (p[0] == 0xf0 && (p[1] & 0x30) == 0) // overlong || (p[0] == 0xf4 && p[1] > 0x8f) || p[0] > 0xf4 // > U+10FFFF ) return false; @@ -146,24 +147,74 @@ write(std::uint8_t const* in, std::size_t size) auto const fail_fast = [&]() { - auto const n = p_ - cp_; - switch(n) + if(cp_[0] < 128) { - default: - BOOST_ASSERT(false); - BOOST_FALLTHROUGH; - case 1: - cp_[1] = 0x81; - BOOST_FALLTHROUGH; - case 2: - cp_[2] = 0x81; - BOOST_FALLTHROUGH; - case 3: - cp_[3] = 0x81; - break; + return false; } - std::uint8_t const* p = cp_; - return ! valid(p); + + const auto& p = cp_; // alias, only to keep this code similar to valid() above + const auto known_only = p_ - cp_; + if (known_only == 1) + { + if((p[0] & 0xe0) == 0xc0) + { + return ((p[0] & 0x1e) == 0); // overlong + } + if((p[0] & 0xf0) == 0xe0) + { + return false; + } + if((p[0] & 0xf8) == 0xf0) + { + return ((p[0] & 0x07) >= 0x05); // invalid F5...FF characters + } + } + else if (known_only == 2) + { + if((p[0] & 0xe0) == 0xc0) + { + return ((p[1] & 0xc0) != 0x80 || + (p[0] & 0x1e) == 0); // overlong + } + if((p[0] & 0xf0) == 0xe0) + { + return ( (p[1] & 0xc0) != 0x80 + || (p[0] == 0xe0 && (p[1] & 0x20) == 0) // overlong + || (p[0] == 0xed && (p[1] & 0x20) == 0x20)); // surrogate + } + if((p[0] & 0xf8) == 0xf0) + { + return ( (p[0] & 0x07) >= 0x05 // invalid F5...FF characters + || (p[1] & 0xc0) != 0x80 + || (p[0] == 0xf0 && (p[1] & 0x30) == 0) // overlong + || (p[0] == 0xf4 && p[1] > 0x8f) || p[0] > 0xf4); // > U+10FFFF + } + } + else if (known_only == 3) + { + if((p[0] & 0xe0) == 0xc0) + { + return ( (p[1] & 0xc0) != 0x80 + || (p[0] & 0x1e) == 0); // overlong + } + if((p[0] & 0xf0) == 0xe0) + { + return ( (p[1] & 0xc0) != 0x80 + || (p[2] & 0xc0) != 0x80 + || (p[0] == 0xe0 && (p[1] & 0x20) == 0) // overlong + || (p[0] == 0xed && (p[1] & 0x20) == 0x20)); // surrogate + //|| (p[0] == 0xef && p[1] == 0xbf && (p[2] & 0xfe) == 0xbe) // U+FFFE or U+FFFF + } + if((p[0] & 0xf8) == 0xf0) + { + return ( (p[0] & 0x07) >= 0x05 // invalid F5...FF characters + || (p[1] & 0xc0) != 0x80 + || (p[2] & 0xc0) != 0x80 + || (p[0] == 0xf0 && (p[1] & 0x30) == 0) // overlong + || (p[0] == 0xf4 && p[1] > 0x8f) || p[0] > 0xf4); // > U+10FFFF + } + } + return true; }; auto const needed = [](std::uint8_t const v) diff --git a/test/beast/websocket/utf8_checker.cpp b/test/beast/websocket/utf8_checker.cpp index 8797657a..924c5202 100644 --- a/test/beast/websocket/utf8_checker.cpp +++ b/test/beast/websocket/utf8_checker.cpp @@ -58,15 +58,9 @@ public: // three byte sequences for(unsigned char c = 224; c < 240; ++c) { - // fail fast utf8_checker u; - if (c == 224) - BEAST_EXPECT(! u.write(&c, 1)); - else - { - BEAST_EXPECT(u.write(&c, 1)); - BEAST_EXPECT(! u.finish()); - } + BEAST_EXPECT(u.write(&c, 1)); + BEAST_EXPECT(! u.finish()); } // four byte sequences @@ -74,13 +68,8 @@ public: { // fail fast utf8_checker u; - if (c == 240) - BEAST_EXPECT(! u.write(&c, 1)); - else - { - BEAST_EXPECT(u.write(&c, 1)); - BEAST_EXPECT(! u.finish()); - } + BEAST_EXPECT(u.write(&c, 1)); + BEAST_EXPECT(! u.finish()); } // invalid lead bytes @@ -167,8 +156,11 @@ public: BEAST_EXPECT(u.write(buf, 3)); BEAST_EXPECT(u.finish()); // Segmented sequence - if (i == 224) - BEAST_EXPECT(! u.write(buf, 1)); + if (i == 224) + { + BEAST_EXPECT(u.write(buf, 1)); + BEAST_EXPECT(!u.finish()); + } else { BEAST_EXPECT(u.write(buf, 1)); @@ -270,8 +262,10 @@ public: } // Segmented sequence second byte invalid - if (i == 224) - BEAST_EXPECT(! u.write(buf, 1)); + if (i == 224) { + BEAST_EXPECT(u.write(buf, 1)); + BEAST_EXPECT(!u.finish()); + } else { BEAST_EXPECT(u.write(buf, 1)); @@ -311,13 +305,8 @@ public: BEAST_EXPECT(u.write(buf, 4)); BEAST_EXPECT(u.finish()); // Segmented sequence - if (i == 240) - BEAST_EXPECT(! u.write(buf, 1)); - else - { - BEAST_EXPECT(u.write(buf, 1)); - BEAST_EXPECT(u.write(&buf[1], 3)); - } + BEAST_EXPECT(u.write(buf, 1)); + BEAST_EXPECT(u.write(&buf[1], 3)); u.reset(); // Segmented sequence BEAST_EXPECT(u.write(buf, 2)); @@ -424,13 +413,9 @@ public: } // Segmented sequence second byte invalid - if (i == 240) - BEAST_EXPECT(! u.write(buf, 1)); - else - { - BEAST_EXPECT(u.write(buf, 1)); - BEAST_EXPECT(! u.write(&buf[1], 1)); - } + BEAST_EXPECT(u.write(buf, 1)); + BEAST_EXPECT(! u.write(&buf[1], 1)); + u.reset(); } @@ -534,6 +519,53 @@ public: } } + void + AutodeskTests() + { + std::vector> const data{ + { 's','t','a','r','t', 0xE0 }, + { 0xA6, 0x81, 'e','n','d' } }; + utf8_checker u; + for(auto const& s : data) + { + std::size_t n = s.size(); + buffers_suffix cb{boost::asio::const_buffer(s.data(), n)}; + multi_buffer b; + while(n) + { + auto const amount = (std::min)(n, std::size_t(3)/*size*/); + b.commit(boost::asio::buffer_copy(b.prepare(amount), cb)); + cb.consume(amount); + n -= amount; + } + BEAST_EXPECT(u.write(b.data())); + } + BEAST_EXPECT(u.finish()); + } + + void + AutobahnTest(std::vector>&& data, std::vector result) + { + BEAST_EXPECT(data.size() == result.size()); + utf8_checker u; + for(std::size_t i = 0; i < data.size(); ++i) + { + auto const& s = data[i]; + + std::size_t n = s.size(); + buffers_suffix cb{boost::asio::const_buffer(s.data(), n)}; + multi_buffer b; + while(n) + { + auto const amount = (std::min)(n, std::size_t(3)/*size*/); + b.commit(boost::asio::buffer_copy(b.prepare(amount), cb)); + cb.consume(amount); + n -= amount; + } + BEAST_EXPECT(u.write(b.data()) == result[i]); + } + } + void run() override { @@ -543,6 +575,17 @@ public: testFourByteSequence(); testWithStreamBuffer(); testBranches(); + AutodeskTests(); + // 6.4.2 + AutobahnTest(std::vector>{ + { 0xCE, 0xBA, 0xE1, 0xBD, 0xB9, 0xCF, 0x83, 0xCE, 0xBC, 0xCE, 0xB5, 0xF4 }, + { 0x90 }, { 0x80, 0x80, 0x65, 0x64, 0x69, 0x74, 0x65, 0x64 } }, + { true, false, false}); + // 6.4.4 + AutobahnTest(std::vector>{ + { 0xCE, 0xBA, 0xE1, 0xBD, 0xB9, 0xCF, 0x83, 0xCE, 0xBC, 0xCE, 0xB5, 0xF4 }, + { 0x90 } }, + { true, false }); } };