C++: clean up numeric literal parsing and add support for n3472.

Separate the messy pp-number parsing from the numeric literal parsing.
The C/C++ preprocessor makes a grown man cry, but at least we have
"proper" literal parsing when we want it, including C++1y binary
literals.

Next step is digit separators (n3781).

Change-Id: Ia069eef454ed5c056f77694a5b8a595d0b76adc4
Reviewed-by: Erik Verbruggen <erik.verbruggen@theqtcompany.com>

src/libs/cplusplus/SimpleLexer.cpp

@@ -41,7 +41,8 @@ using namespace CPlusPlus;
 SimpleLexer::SimpleLexer()
     : _lastState(0),
       _skipComments(false),
-      _endedJoined(false)
+      _endedJoined(false),
+      _ppMode(false)
 {}
 
 SimpleLexer::~SimpleLexer()
@@ -73,6 +74,7 @@ Tokens SimpleLexer::operator()(const QString &text, int state)
     Lexer lex(firstChar, lastChar);
     lex.setLanguageFeatures(_languageFeatures);
     lex.setStartWithNewline(true);
+    lex.setPreprocessorMode(_ppMode);
 
     if (! _skipComments)
         lex.setScanCommentTokens(true);

src/libs/cplusplus/SimpleLexer.h

@@ -51,6 +51,9 @@ public:
     bool skipComments() const;
     void setSkipComments(bool skipComments);
 
+    void setPreprocessorMode(bool ppMode)
+    { _ppMode = ppMode; }
+
     LanguageFeatures languageFeatures() const { return _languageFeatures; }
     void setLanguageFeatures(LanguageFeatures features) { _languageFeatures = features; }
 
@@ -74,6 +77,7 @@ private:
     LanguageFeatures _languageFeatures;
     bool _skipComments: 1;
     bool _endedJoined: 1;
+    bool _ppMode: 1;
 };
 
 } // namespace CPlusPlus

src/libs/cplusplus/pp-engine.cpp

@@ -401,6 +401,9 @@ protected:
             const char *end = spell + len;
             char *vend = const_cast<char *>(end);
             _value.set_long(strtol(spell, &vend, 0));
+            // TODO: if (vend != end) error(NaN)
+            // TODO: binary literals
+            // TODO: float literals
             ++(*_lex);
         } else if (isTokenDefined()) {
             ++(*_lex);
@@ -1388,6 +1391,7 @@ void Preprocessor::preprocess(const QString &fileName, const QByteArray &source,
     m_state.m_lexer = new Lexer(source.constBegin(), source.constEnd());
     m_state.m_lexer->setScanKeywords(false);
     m_state.m_lexer->setScanAngleStringLiteralTokens(false);
+    m_state.m_lexer->setPreprocessorMode(true);
     if (m_keepComments)
         m_state.m_lexer->setScanCommentTokens(true);
     m_state.m_result = result;
@@ -1803,6 +1807,7 @@ const PPToken Preprocessor::evalExpression(PPToken *tk, Value &result)
     PPToken lastConditionToken;
     const QByteArray expanded = expand(tk, &lastConditionToken);
     Lexer lexer(expanded.constData(), expanded.constData() + expanded.size());
+    lexer.setPreprocessorMode(true);
     std::vector<Token> buf;
     Token t;
     do {