CPlusPlus: Fix lexer crash

At some point in the preprocessing stage, some tokens get "squeezed", so
their associated string no longer refers to the document content, but
only to their own symbol. However, there is at least one context that
operated under the assumption that the token's offset still pointed into
the "global" string. As the token's offset is zero after parsing, this
lead to the same piece of code being preprocessed in an infinite loop.

Fixes: QTCREATORBUG-19525
Change-Id: I231ba51811cfa0b5c6dfe7f75fe0384472252c6f
Reviewed-by: Christian Stenger <christian.stenger@qt.io>

src/libs/cplusplus/PPToken.cpp

@@ -54,6 +54,7 @@ void PPToken::squeezeSource()
     if (hasSource()) {
         m_src = m_src.mid(byteOffset, f.bytes);
         m_src.squeeze();
+        m_originalOffset = byteOffset;
         byteOffset = 0;
         utf16charOffset = 0;
     }

src/libs/cplusplus/PPToken.h

@@ -129,8 +129,19 @@ public:
     ByteArrayRef asByteArrayRef() const
     { return ByteArrayRef(&m_src, byteOffset, bytes()); }
 
+    unsigned originalOffset() const
+    { return m_originalOffset != -1 ? m_originalOffset : byteOffset; }
+
 private:
     QByteArray m_src;
+
+    // TODO: We may or may not be able to get rid of this member. In order to find out,
+    //       all code calling this class' accessors (including the parent class'
+    //       bytes* and utf16* functions) has to be looked at. Essentially, it boils
+    //       down to whether there are contexts where an object of this class is used
+    //       and the original "global" string is no longer available. (If not, then the
+    //       m_src member would also not be needed.)
+    int m_originalOffset = -1;
 };
 
 } // namespace Internal

src/libs/cplusplus/pp-engine.cpp

@@ -1856,7 +1856,7 @@ void Preprocessor::handleDefineDirective(PPToken *tk)
 QByteArray Preprocessor::expand(PPToken *tk, PPToken *lastConditionToken)
 {
     unsigned line = tk->lineno;
-    unsigned bytesBegin = tk->bytesBegin();
+    unsigned bytesBegin = tk->originalOffset();
     unsigned utf16charsBegin = tk->utf16charsBegin();
     PPToken lastTk;
     while (isContinuationToken(*tk)) {

tests/auto/cplusplus/preprocessor/tst_preprocessor.cpp

@@ -405,6 +405,7 @@ private slots:
     void multi_byte_code_point_in_expansion();
     void trigraph();
     void nested_arguments_expansion();
+    void preprocessorSymbolsAsMacroArguments();
 };
 
 // Remove all #... lines, and 'simplify' string, to allow easily comparing the result
@@ -2099,6 +2100,20 @@ void tst_Preprocessor::nested_arguments_expansion()
     QVERIFY(prep.contains(output));
 }
 
+void tst_Preprocessor::preprocessorSymbolsAsMacroArguments()
+{
+    Environment env;
+    Preprocessor preprocess(nullptr, &env);
+    const QByteArray input =
+            "#define IFGEN(if, endif) if (1 == 0) endif\n"
+            "int main()\n"
+            "{\n"
+            "IFGEN(#if, #endif)\n"
+            "return 0;\n"
+            "}\n";
+    QVERIFY(preprocess.run(QLatin1String("<stdin>"), input).startsWith("# 1 \"<stdin>\"\n"));
+}
+
 void tst_Preprocessor::excessive_nesting()
 {
     Environment env;