From 9f104b0664a75b0094b7b775ef71370b57238af5 Mon Sep 17 00:00:00 2001 From: jzmaddock Date: Thu, 2 Nov 2017 11:16:02 +0000 Subject: [PATCH 1/5] Regex: Streamline travis builds further. --- .travis.yml | 209 ++++++---------------------------------------------- 1 file changed, 23 insertions(+), 186 deletions(-) diff --git a/.travis.yml b/.travis.yml index 87a39402..7a124337 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ matrix: env: TOOLSET=gcc COMPILER=g++ CXXSTD=c++03 - os: linux - env: TOOLSET=gcc COMPILER=g++-4.7 CXXSTD=c++03 + env: TOOLSET=gcc COMPILER=g++-4.7 CXXSTD=03,11 addons: apt: packages: @@ -40,16 +40,7 @@ matrix: - ubuntu-toolchain-r-test - os: linux - env: TOOLSET=gcc COMPILER=g++-4.7 CXXSTD=c++11 - addons: - apt: - packages: - - g++-4.7 - sources: - - ubuntu-toolchain-r-test - - - os: linux - env: TOOLSET=gcc COMPILER=g++-4.8 CXXSTD=c++03 + env: TOOLSET=gcc COMPILER=g++-4.8 CXXSTD=03,11 addons: apt: packages: @@ -58,16 +49,7 @@ matrix: - ubuntu-toolchain-r-test - os: linux - env: TOOLSET=gcc COMPILER=g++-4.8 CXXSTD=c++11 - addons: - apt: - packages: - - g++-4.8 - sources: - - ubuntu-toolchain-r-test - - - os: linux - env: TOOLSET=gcc COMPILER=g++-4.9 CXXSTD=c++03 + env: TOOLSET=gcc COMPILER=g++-4.9 CXXSTD=03,11 addons: apt: packages: @@ -76,16 +58,7 @@ matrix: - ubuntu-toolchain-r-test - os: linux - env: TOOLSET=gcc COMPILER=g++-4.9 CXXSTD=c++11 - addons: - apt: - packages: - - g++-4.9 - sources: - - ubuntu-toolchain-r-test - - - os: linux - env: TOOLSET=gcc COMPILER=g++-5 CXXSTD=c++03 + env: TOOLSET=gcc COMPILER=g++-5 CXXSTD=03,11,14 addons: apt: packages: @@ -94,25 +67,7 @@ matrix: - ubuntu-toolchain-r-test - os: linux - env: TOOLSET=gcc COMPILER=g++-5 CXXSTD=c++11 - addons: - apt: - packages: - - g++-5 - sources: - - ubuntu-toolchain-r-test - - - os: linux - env: TOOLSET=gcc COMPILER=g++-5 CXXSTD=c++14 - addons: - apt: - packages: - - g++-5 - sources: - - ubuntu-toolchain-r-test - - - os: linux - env: TOOLSET=gcc COMPILER=g++-6 CXXSTD=c++03 + env: TOOLSET=gcc COMPILER=g++-6 CXXSTD=03,11 addons: apt: packages: @@ -121,7 +76,7 @@ matrix: - ubuntu-toolchain-r-test - os: linux - env: TOOLSET=gcc COMPILER=g++-6 CXXSTD=c++11 + env: TOOLSET=gcc COMPILER=g++-6 CXXSTD=14,1z addons: apt: packages: @@ -130,7 +85,7 @@ matrix: - ubuntu-toolchain-r-test - os: linux - env: TOOLSET=gcc COMPILER=g++-6 CXXSTD=c++14 + env: TOOLSET=gcc COMPILER=g++-6 CXXSTD=03,11 CXXSTD_DIALECT=cxxstd-dialect=gnu addons: apt: packages: @@ -139,7 +94,7 @@ matrix: - ubuntu-toolchain-r-test - os: linux - env: TOOLSET=gcc COMPILER=g++-6 CXXSTD=c++1z + env: TOOLSET=gcc COMPILER=g++-6 CXXSTD=14,1z CXXSTD_DIALECT=cxxstd-dialect=gnu addons: apt: packages: @@ -148,49 +103,10 @@ matrix: - ubuntu-toolchain-r-test - os: linux - env: TOOLSET=gcc COMPILER=g++-6 CXXSTD=gnu++03 - addons: - apt: - packages: - - g++-6 - sources: - - ubuntu-toolchain-r-test + env: TOOLSET=clang COMPILER=clang++ CXXSTD=03,11 - os: linux - env: TOOLSET=gcc COMPILER=g++-6 CXXSTD=gnu++11 - addons: - apt: - packages: - - g++-6 - sources: - - ubuntu-toolchain-r-test - - - os: linux - env: TOOLSET=gcc COMPILER=g++-6 CXXSTD=gnu++14 - addons: - apt: - packages: - - g++-6 - sources: - - ubuntu-toolchain-r-test - - - os: linux - env: TOOLSET=gcc COMPILER=g++-6 CXXSTD=gnu++1z - addons: - apt: - packages: - - g++-6 - sources: - - ubuntu-toolchain-r-test - - - os: linux - env: TOOLSET=clang COMPILER=clang++ CXXSTD=c++03 - - - os: linux - env: TOOLSET=clang COMPILER=clang++ CXXSTD=c++11 - - - os: linux - env: TOOLSET=clang COMPILER=clang++-3.5 CXXSTD=c++03 + env: TOOLSET=clang COMPILER=clang++-3.5 CXXSTD=03,11 addons: apt: packages: @@ -200,17 +116,7 @@ matrix: - llvm-toolchain-precise-3.5 - os: linux - env: TOOLSET=clang COMPILER=clang++-3.5 CXXSTD=c++11 - addons: - apt: - packages: - - clang-3.5 - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.5 - - - os: linux - env: TOOLSET=clang COMPILER=clang++-3.6 CXXSTD=c++03 + env: TOOLSET=clang COMPILER=clang++-3.6 CXXSTD=03,11 addons: apt: packages: @@ -220,17 +126,7 @@ matrix: - llvm-toolchain-precise-3.6 - os: linux - env: TOOLSET=clang COMPILER=clang++-3.6 CXXSTD=c++11 - addons: - apt: - packages: - - clang-3.6 - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.6 - - - os: linux - env: TOOLSET=clang COMPILER=clang++-3.7 CXXSTD=c++03 + env: TOOLSET=clang COMPILER=clang++-3.7 CXXSTD=03,11 addons: apt: packages: @@ -240,17 +136,7 @@ matrix: - llvm-toolchain-precise-3.7 - os: linux - env: TOOLSET=clang COMPILER=clang++-3.7 CXXSTD=c++11 - addons: - apt: - packages: - - clang-3.7 - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.7 - - - os: linux - env: TOOLSET=clang COMPILER=clang++-3.8 CXXSTD=c++03 + env: TOOLSET=clang COMPILER=clang++-3.8 CXXSTD=03,11 addons: apt: packages: @@ -260,17 +146,7 @@ matrix: - llvm-toolchain-precise-3.8 - os: linux - env: TOOLSET=clang COMPILER=clang++-3.8 CXXSTD=c++11 - addons: - apt: - packages: - - clang-3.8 - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.8 - - - os: linux - env: TOOLSET=clang COMPILER=clang++-3.8 CXXSTD=c++14 + env: TOOLSET=clang COMPILER=clang++-3.8 CXXSTD=14,1z addons: apt: packages: @@ -281,18 +157,7 @@ matrix: - llvm-toolchain-precise-3.8 - os: linux - env: TOOLSET=clang COMPILER=clang++-3.8 CXXSTD=c++1z - addons: - apt: - packages: - - clang-3.8 - - libstdc++-4.9-dev - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.8 - - - os: linux - env: TOOLSET=clang COMPILER=clang++-3.9 CXXSTD=c++03 + env: TOOLSET=clang COMPILER=clang++-3.9 CXXSTD=03,11 addons: apt: packages: @@ -303,29 +168,7 @@ matrix: - llvm-toolchain-precise-3.9 - os: linux - env: TOOLSET=clang COMPILER=clang++-3.9 CXXSTD=c++11 - addons: - apt: - packages: - - clang-3.9 - - libstdc++-4.9-dev - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.9 - - - os: linux - env: TOOLSET=clang COMPILER=clang++-3.9 CXXSTD=c++14 - addons: - apt: - packages: - - clang-3.9 - - libstdc++-4.9-dev - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.9 - - - os: linux - env: TOOLSET=clang COMPILER=clang++-3.9 CXXSTD=c++1z + env: TOOLSET=clang COMPILER=clang++-3.9 CXXSTD=14,1z addons: apt: packages: @@ -336,19 +179,11 @@ matrix: - llvm-toolchain-precise-3.9 - os: osx - env: TOOLSET=clang COMPILER=clang++ CXXSTD=c++03 + env: TOOLSET=clang COMPILER=clang++ CXXSTD=03,11 osx_image: xcode8.3 - os: osx - env: TOOLSET=clang COMPILER=clang++ CXXSTD=c++11 - osx_image: xcode8.3 - - - os: osx - env: TOOLSET=clang COMPILER=clang++ CXXSTD=c++14 - osx_image: xcode8.3 - - - os: osx - env: TOOLSET=clang COMPILER=clang++ CXXSTD=c++1z + env: TOOLSET=clang COMPILER=clang++ CXXSTD=14,1z osx_image: xcode8.3 - os: osx @@ -426,9 +261,11 @@ install: script: - |- - echo "using $TOOLSET : : $COMPILER : -std=$CXXSTD ;" > ~/user-config.jam - - (cd libs/config/test && ../../../b2 config_info_travis_install toolset=$TOOLSET && ./config_info_travis) - - ./b2 -j3 libs/regex/test toolset=$TOOLSET + echo "using $TOOLSET : : $COMPILER ;" > ~/user-config.jam + - IFS=',' + - for CXXLOCAL in $CXXSTD; do (cd libs/config/test && ../../../b2 config_info_travis_install toolset=$TOOLSET cxxstd=$CXXLOCAL $CXXSTD_DIALECT && echo With Standard Version $CXXLOCAL && ./config_info_travis && rm ./config_info_travis) done + - unset IFS + - ./b2 -j3 libs/regex/test toolset=$TOOLSET cxxstd=$CXXSTD $CXXSTD_DIALECT notifications: email: From 6246f5ec5183b5cf32fb488288ffc86c5b9ee36b Mon Sep 17 00:00:00 2001 From: jzmaddock Date: Thu, 9 Nov 2017 11:23:54 +0000 Subject: [PATCH 2/5] Regex: Set a limit on how many recursions we allow. --- include/boost/regex/v4/perl_matcher.hpp | 4 +++- .../regex/v4/perl_matcher_non_recursive.hpp | 22 ++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/include/boost/regex/v4/perl_matcher.hpp b/include/boost/regex/v4/perl_matcher.hpp index 198acc27..1182b775 100644 --- a/include/boost/regex/v4/perl_matcher.hpp +++ b/include/boost/regex/v4/perl_matcher.hpp @@ -378,7 +378,7 @@ public: BidiIterator l_base) : m_result(what), base(first), last(end), position(first), backstop(l_base), re(e), traits_inst(e.get_traits()), - m_independent(false), next_count(&rep_obj), rep_obj(&next_count) + m_independent(false), next_count(&rep_obj), rep_obj(&next_count), m_recursions(0) { construct_init(e, f); } @@ -566,6 +566,8 @@ private: bool m_unwound_alt; // We are unwinding a commit - used by independent subs to determine whether to stop there or carry on unwinding: //bool m_unwind_commit; + // Recursion limit: + unsigned m_recursions; #endif // these operations aren't allowed, so are declared private, diff --git a/include/boost/regex/v4/perl_matcher_non_recursive.hpp b/include/boost/regex/v4/perl_matcher_non_recursive.hpp index 789226ba..3d89749d 100644 --- a/include/boost/regex/v4/perl_matcher_non_recursive.hpp +++ b/include/boost/regex/v4/perl_matcher_non_recursive.hpp @@ -143,6 +143,15 @@ struct saved_change_case : public saved_state saved_change_case(bool c) : saved_state(18), icase(c) {} }; +struct incrementer +{ + incrementer(unsigned* pu) : m_pu(pu) { ++*m_pu; } + ~incrementer() { --*m_pu; } + bool operator > (unsigned i) { return *m_pu > i; } +private: + unsigned* m_pu; +}; + template bool perl_matcher::match_all_states() { @@ -187,7 +196,9 @@ bool perl_matcher::match_all_states() &perl_matcher::match_commit, &perl_matcher::match_then, }; - + incrementer inc(&m_recursions); + if(inc > 80) + raise_error(traits_inst, regex_constants::error_complexity); push_recursion_stopper(); do{ while(pstate) @@ -1762,8 +1773,11 @@ bool perl_matcher::unwind_non_greedy_repeat(boo template bool perl_matcher::unwind_recursion(bool r) { + // We are backtracking back inside a recursion, need to push the info + // back onto the recursion stack, and do so unconditionally, otherwise + // we can get mismatched pushes and pops... saved_recursion* pmp = static_cast*>(m_backup_state); - if(!r) + if (!r) { recursion_stack.push_back(recursion_info()); recursion_stack.back().idx = pmp->recursion_id; @@ -1780,8 +1794,10 @@ bool perl_matcher::unwind_recursion(bool r) template bool perl_matcher::unwind_recursion_pop(bool r) { + // Backtracking out of a recursion, we must pop state off the recursion + // stack unconditionally to ensure matched pushes and pops: saved_state* pmp = static_cast(m_backup_state); - if(!r) + if (!r) { *m_presult = recursion_stack.back().results; position = recursion_stack.back().location_of_start; From e94f079215395ac2d36932a3995f6081538fe956 Mon Sep 17 00:00:00 2001 From: jzmaddock Date: Thu, 9 Nov 2017 11:31:48 +0000 Subject: [PATCH 3/5] Regex: fix locale specific tests so they do all pass now! --- test/regress/test_locale.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/regress/test_locale.cpp b/test/regress/test_locale.cpp index 13aa85b9..692d83e6 100644 --- a/test/regress/test_locale.cpp +++ b/test/regress/test_locale.cpp @@ -191,8 +191,11 @@ void test_en_locale(const char* name, boost::uint32_t lcid) // causes these tests to crash (pointer overrun in std::collate::do_transform). TEST_REGEX_SEARCH_L("[a-z]+", perl|::boost::regex_constants::collate, "\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf8\xf9\xfa\xfb\xfc", match_default, make_array(0, 28, -2, -2)); TEST_REGEX_SEARCH_L("[a-z]+", perl|::boost::regex_constants::collate, "\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd8\xd9\xda\xdb\xdc", match_default, make_array(1, 28, -2, -2)); - // and equivalence classes: - TEST_REGEX_SEARCH_L("[[=a=]]+", perl, "aA\xe0\xe1\xe2\xe3\xe4\xe5\xc0\xc1\xc2\xc3\xc4\xc5", match_default, make_array(0, 14, -2, -2)); + if (lcid != 0x09) + { + // and equivalence classes, these fail for locale "en" but pass for "en_UK" and "en_US": + TEST_REGEX_SEARCH_L("[[=a=]]+", perl, "aA\xe0\xe1\xe2\xe3\xe4\xe5\xc0\xc1\xc2\xc3\xc4\xc5", match_default, make_array(0, 14, -2, -2)); + } // case mapping: TEST_REGEX_SEARCH_L("[A-Z]+", perl|icase|::boost::regex_constants::collate, "\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf8\xf9\xfa\xfb\xfc", match_default, make_array(0, 28, -2, -2)); TEST_REGEX_SEARCH_L("[a-z]+", perl|icase|::boost::regex_constants::collate, "\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd8\xd9\xda\xdb\xdc", match_default, make_array(1, 28, -2, -2)); From 1480e33dc6196caf726d6c4ba4a5b3f97b8a02ac Mon Sep 17 00:00:00 2001 From: jzmaddock Date: Thu, 9 Nov 2017 18:13:16 +0000 Subject: [PATCH 4/5] Regex: Fix recursive algorithm builds. --- include/boost/regex/v4/perl_matcher.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/boost/regex/v4/perl_matcher.hpp b/include/boost/regex/v4/perl_matcher.hpp index 1182b775..a696e5d9 100644 --- a/include/boost/regex/v4/perl_matcher.hpp +++ b/include/boost/regex/v4/perl_matcher.hpp @@ -378,7 +378,10 @@ public: BidiIterator l_base) : m_result(what), base(first), last(end), position(first), backstop(l_base), re(e), traits_inst(e.get_traits()), - m_independent(false), next_count(&rep_obj), rep_obj(&next_count), m_recursions(0) + m_independent(false), next_count(&rep_obj), rep_obj(&next_count) +#ifdef BOOST_REGEX_NON_RECURSIVE + , m_recursions(0) +#endif { construct_init(e, f); } From 2608372174ef271cb31abda18c6fe21e0ba9f54e Mon Sep 17 00:00:00 2001 From: jzmaddock Date: Fri, 10 Nov 2017 08:08:27 +0000 Subject: [PATCH 5/5] Regex: Fix travis.yml usage of cxxstd=. --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7a124337..df49941a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ matrix: include: - os: linux - env: TOOLSET=gcc COMPILER=g++ CXXSTD=c++03 + env: TOOLSET=gcc COMPILER=g++ CXXSTD=03 - os: linux env: TOOLSET=gcc COMPILER=g++-4.7 CXXSTD=03,11 @@ -187,23 +187,23 @@ matrix: osx_image: xcode8.3 - os: osx - env: TOOLSET=clang COMPILER=clang++ CXXSTD=c++11 + env: TOOLSET=clang COMPILER=clang++ CXXSTD=11 osx_image: xcode8.2 - os: osx - env: TOOLSET=clang COMPILER=clang++ CXXSTD=c++11 + env: TOOLSET=clang COMPILER=clang++ CXXSTD=11 osx_image: xcode8.1 - os: osx - env: TOOLSET=clang COMPILER=clang++ CXXSTD=c++11 + env: TOOLSET=clang COMPILER=clang++ CXXSTD=11 osx_image: xcode8.0 - os: osx - env: TOOLSET=clang COMPILER=clang++ CXXSTD=c++11 + env: TOOLSET=clang COMPILER=clang++ CXXSTD=11 osx_image: xcode7.3 - os: osx - env: TOOLSET=clang COMPILER=clang++ CXXSTD=c++11 + env: TOOLSET=clang COMPILER=clang++ CXXSTD=11 osx_image: xcode6.4 install: