C++: Support single quote digit separator in integer literals

C++14 supports the use of single quotes inserted between integer digits
as a separator.  Updates the built-in C++ code model to recognize such
quotes.  This fixes highlighting and indentation issues.

Change-Id: Ic35ce93060b96700a11d108dce1f3cf6c4543632
Fixes: QTCREATORBUG-14939
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@qt.io>

src/libs/3rdparty/cplusplus/Lexer.cpp

@@ -954,7 +954,8 @@ void Lexer::scanNumericLiteral(Token *tok)
             yyinp();
             while (std::isdigit(_yychar) ||
                    (_yychar >= 'a' && _yychar <= 'f') ||
-                   (_yychar >= 'A' && _yychar <= 'F')) {
+                   (_yychar >= 'A' && _yychar <= 'F') ||
+                   ((_yychar == '\'') && _languageFeatures.cxx14Enabled)) {
                 yyinp();
             }
             if (!scanOptionalIntegerSuffix())
@@ -962,7 +963,8 @@ void Lexer::scanNumericLiteral(Token *tok)
             goto theEnd;
         } else if (_yychar == 'b' || _yychar == 'B') { // see n3472
             yyinp();
-            while (_yychar == '0' || _yychar == '1')
+            while (_yychar == '0' || _yychar == '1' ||
+                   ((_yychar == '\'') && _languageFeatures.cxx14Enabled))
                 yyinp();
             if (!scanOptionalIntegerSuffix())
                 scanOptionalUserDefinedLiteral(tok);
@@ -970,7 +972,8 @@ void Lexer::scanNumericLiteral(Token *tok)
         } else if (_yychar >= '0' && _yychar <= '7') {
             do {
                 yyinp();
-            } while (_yychar >= '0' && _yychar <= '7');
+            } while ((_yychar >= '0' && _yychar <= '7') ||
+                     ((_yychar == '\'') && _languageFeatures.cxx14Enabled));
             if (!scanOptionalIntegerSuffix())
                 scanOptionalUserDefinedLiteral(tok);
             goto theEnd;
@@ -989,7 +992,8 @@ void Lexer::scanNumericLiteral(Token *tok)
             if (scanExponentPart() && !scanOptionalFloatingSuffix())
                 scanOptionalUserDefinedLiteral(tok);
             break;
-        } else if (std::isdigit(_yychar)) {
+        } else if (std::isdigit(_yychar) ||
+                   ((_yychar == '\'') && _languageFeatures.cxx14Enabled)) {
             yyinp();
         } else {
             if (!scanOptionalIntegerSuffix())

src/libs/3rdparty/cplusplus/Token.h

@@ -437,6 +437,7 @@ struct LanguageFeatures
             unsigned int qtKeywordsEnabled : 1; // If Qt is used but QT_NO_KEYWORDS defined
             unsigned int cxxEnabled : 1;
             unsigned int cxx11Enabled : 1;
+            unsigned int cxx14Enabled : 1;
             unsigned int objCEnabled : 1;
             unsigned int c99Enabled : 1;
         };

src/libs/cplusplus/MatchingText.cpp

@@ -145,6 +145,7 @@ static LanguageFeatures languageFeatures()
     features.qtKeywordsEnabled = false;
     features.qtMocRunEnabled = false;
     features.cxx11Enabled = true;
+    features.cxx14Enabled = true;
     features.cxxEnabled = true;
     features.c99Enabled = true;
     features.objCEnabled = true;

src/plugins/cpptools/cppcodeformatter.cpp

@@ -1054,6 +1054,8 @@ int CodeFormatter::tokenizeBlock(const QTextBlock &block, bool *endedJoined)
     features.qtKeywordsEnabled = true;
     features.cxxEnabled = true;
     features.objCEnabled = true;
+    features.cxx11Enabled = true;
+    features.cxx14Enabled = true;
 
     SimpleLexer tokenize;
     tokenize.setLanguageFeatures(features);

src/plugins/cpptools/projectpart.cpp

@@ -38,6 +38,7 @@ void ProjectPart::updateLanguageFeatures()
     const bool hasCxx = languageVersion >= Utils::LanguageVersion::CXX98;
     const bool hasQt = hasCxx && qtVersion != NoQt;
     languageFeatures.cxx11Enabled = languageVersion >= Utils::LanguageVersion::CXX11;
+    languageFeatures.cxx14Enabled = languageVersion >= Utils::LanguageVersion::CXX14;
     languageFeatures.cxxEnabled = hasCxx;
     languageFeatures.c99Enabled = languageVersion >= Utils::LanguageVersion::C99;
     languageFeatures.objCEnabled = languageExtensions.testFlag(Utils::LanguageExtension::ObjectiveC);

tests/auto/cplusplus/lexer/tst_lexer.cpp

@@ -277,7 +277,10 @@ void tst_SimpleLexer::literals()
     QFETCH(QByteArray, source);
     QFETCH(TokenKindList, expectedTokenKindList);
 
-    run(source, toTokens(expectedTokenKindList), false, CompareKind);
+    LanguageFeatures features;
+    features.cxx14Enabled = true;
+    run(source, toTokens(expectedTokenKindList), false, CompareKind, false,
+        features);
 }
 
 void tst_SimpleLexer::literals_data()
@@ -320,14 +323,18 @@ void tst_SimpleLexer::literals_data()
     source = // these are all the same
             "42\n"
             "0b101010u\n"
+            "0b101'010u\n"
             "052ll\n"
+            "0'5'2ll\n"
             "0x2aL\n"
+            "0x2'aL\n"
             "123FOO\n"
             "0xfOo\n"
             "33_\n"
             ;
     expectedTokenKindList =
             TokenKindList() << T_NUMERIC_LITERAL << T_NUMERIC_LITERAL << T_NUMERIC_LITERAL
+                            << T_NUMERIC_LITERAL << T_NUMERIC_LITERAL << T_NUMERIC_LITERAL
                             << T_NUMERIC_LITERAL << T_ERROR << T_ERROR << T_ERROR
                                ;
     QTest::newRow("integer-literals") << source << expectedTokenKindList;