C++: Fix highlighting after "invalid code"

For the semantic info document we do not expand function like macros and
because of that certain macro invocations lead to invalid code that we
need to handle, e.g.:

	Q_GLOBAL_STATIC(CppTools::SymbolFinder, symbolFinder)
	class Foo {};

This change makes parsing Foo in the semantic info document successfully
again, which affects highlighting of that class.

Change-Id: I389265ac64d3f0b8b8f406d38fa58d78820b14ba
Reviewed-by: Erik Verbruggen <erik.verbruggen@theqtcompany.com>

src/libs/3rdparty/cplusplus/Parser.cpp

@@ -66,8 +66,6 @@ public:
 
 int DebugRule::depth = 0;
 
-const int declarationsInRowAllowedToFail = 2;
-
 inline bool lookAtAssignmentOperator(int tokenKind)
 {
     switch (tokenKind) {
@@ -267,12 +265,13 @@ inline void debugPrintCheckCache(bool) {}
     return true; \
 }
 
-Parser::Parser(TranslationUnit *unit)
+Parser::Parser(TranslationUnit *unit, int retryParseDeclarationLimit)
     : _translationUnit(unit),
       _control(unit->control()),
       _pool(unit->memoryPool()),
       _languageFeatures(unit->languageFeatures()),
       _tokenIndex(1),
+      _retryParseDeclarationLimit(retryParseDeclarationLimit),
       _templateArguments(0),
       _inFunctionBody(false),
       _inExpressionStatement(false),
@@ -657,7 +656,7 @@ bool Parser::parseTranslationUnit(TranslationUnitAST *&node)
         } else {
             error(start_declaration, "expected a declaration");
             rewind(start_declaration + 1);
-            if (++declarationsInRowFailedToParse == declarationsInRowAllowedToFail)
+            if (++declarationsInRowFailedToParse == _retryParseDeclarationLimit)
                 skipUntilAfterSemicolonOrRightBrace();
             else
                 skipUntilDeclaration();
@@ -825,7 +824,7 @@ bool Parser::parseLinkageBody(DeclarationAST *&node)
             } else {
                 error(start_declaration, "expected a declaration");
                 rewind(start_declaration + 1);
-                if (++declarationsInRowFailedToParse == declarationsInRowAllowedToFail)
+                if (++declarationsInRowFailedToParse == _retryParseDeclarationLimit)
                     skipUntilAfterSemicolonOrRightBrace();
                 else
                     skipUntilDeclaration();

src/libs/3rdparty/cplusplus/Parser.h

@@ -33,7 +33,7 @@ namespace CPlusPlus {
 class CPLUSPLUS_EXPORT Parser
 {
 public:
-    Parser(TranslationUnit *translationUnit);
+    Parser(TranslationUnit *translationUnit, int retryParseDeclarationLimit);
     ~Parser();
 
     bool parseTranslationUnit(TranslationUnitAST *&node);
@@ -317,6 +317,7 @@ private:
     MemoryPool *_pool;
     LanguageFeatures _languageFeatures;
     unsigned _tokenIndex;
+    int _retryParseDeclarationLimit;
     bool _templateArguments: 1;
     bool _inFunctionBody: 1;
     bool _inExpressionStatement: 1;

src/libs/3rdparty/cplusplus/TranslationUnit.cpp

@@ -49,7 +49,8 @@ TranslationUnit::TranslationUnit(Control *control, const StringLiteral *fileId)
       _lastSourceChar(0),
       _pool(0),
       _ast(0),
-      _flags(0)
+      _flags(0),
+      _retryParseDeclarationLimit(defaultRetryParseDeclarationLimit())
 {
     _tokens = new std::vector<Token>();
     _comments = new std::vector<Token>();
@@ -299,7 +300,7 @@ bool TranslationUnit::parse(ParseMode mode)
 
     f._parsed = true;
 
-    Parser parser(this);
+    Parser parser(this, _retryParseDeclarationLimit);
     bool parsed = false;
 
     switch (mode) {

src/libs/3rdparty/cplusplus/TranslationUnit.h

@@ -149,6 +149,9 @@ public:
     LanguageFeatures languageFeatures() const { return _languageFeatures; }
     void setLanguageFeatures(LanguageFeatures features) { _languageFeatures = features; }
 
+    static int defaultRetryParseDeclarationLimit() { return 2; }
+    void setRetryParseDeclarationLimit(int limit) { _retryParseDeclarationLimit = limit; }
+
 private:
     struct PPLine {
         unsigned utf16charOffset;
@@ -210,6 +213,8 @@ private:
         Flags f;
     };
     LanguageFeatures _languageFeatures;
+
+    int _retryParseDeclarationLimit;
 };
 
 } // namespace CPlusPlus

src/libs/cplusplus/CppDocument.cpp

@@ -623,6 +623,13 @@ void Document::tokenize()
     _translationUnit->tokenize();
 }
 
+void Document::setRetryHarderToParseDeclarations(bool yesno)
+{
+    _translationUnit->setRetryParseDeclarationLimit(
+                yesno ? 1000
+                      : TranslationUnit::defaultRetryParseDeclarationLimit());
+}
+
 bool Document::isParsed() const
 {
     return _translationUnit->isParsed();

src/libs/cplusplus/CppDocument.h

@@ -127,6 +127,7 @@ public:
     bool isTokenized() const;
     void tokenize();
 
+    void setRetryHarderToParseDeclarations(bool yesno);
     bool isParsed() const;
     bool parse(ParseMode mode = ParseTranlationUnit);
 

src/plugins/cpptools/cppsemanticinfoupdater.cpp

@@ -125,6 +125,7 @@ SemanticInfo SemanticInfoUpdaterPrivate::update(const SemanticInfo::Source &sour
     Document::Ptr doc = newSemanticInfo.snapshot.preprocessedDocument(source.code, source.fileName);
     if (processor)
         doc->control()->setTopLevelDeclarationProcessor(processor);
+    doc->setRetryHarderToParseDeclarations(true);
     doc->check();
     if (processor && processor->isCanceled())
         newSemanticInfo.complete = false;

tests/auto/cplusplus/ast/tst_ast.cpp

@@ -51,7 +51,9 @@ public:
                            TranslationUnit::ParseMode mode,
                            bool blockErrors = false,
                            bool qtMocRun = false,
-                           bool cxx11Enabled = false)
+                           bool cxx11Enabled = false,
+                           int retryParseDeclarationLimit
+                            = TranslationUnit::defaultRetryParseDeclarationLimit())
     {
         const StringLiteral *fileId = control.stringLiteral("<stdin>");
         LanguageFeatures features;
@@ -62,6 +64,7 @@ public:
         features.qtMocRunEnabled = qtMocRun;
         TranslationUnit *unit = new TranslationUnit(&control, fileId);
         unit->setLanguageFeatures(features);
+        unit->setRetryParseDeclarationLimit(retryParseDeclarationLimit);
         unit->setSource(source.constData(), source.length());
         unit->blockErrors(blockErrors);
         unit->parse(mode);
@@ -199,6 +202,7 @@ private slots:
     void unnamed_class();
     void unnamed_class_data();
     void expensiveExpression();
+    void invalidCode();
 };
 
 void tst_AST::gcc_attributes_1()
@@ -1856,6 +1860,27 @@ void tst_AST::expensiveExpression()
     QVERIFY(unit->ast());
 }
 
+void tst_AST::invalidCode()
+{
+    const QByteArray invalidCode = "static inValidLine()\n"
+                                   "class Foo {};\n";
+
+    QSharedPointer<TranslationUnit> unit(parse(invalidCode, TranslationUnit::ParseTranlationUnit,
+                                               false, false, false, 1000));
+    QVERIFY(unit->ast());
+    TranslationUnitAST *unitAST = unit->ast()->asTranslationUnit();
+    QVERIFY(unitAST->declaration_list);
+    QVERIFY(unitAST->declaration_list->value);
+    SimpleDeclarationAST *simpleDecl = unitAST->declaration_list->value->asSimpleDeclaration();
+    QVERIFY(simpleDecl);
+    QVERIFY(simpleDecl->decl_specifier_list);
+    QVERIFY(simpleDecl->decl_specifier_list->value);
+    ClassSpecifierAST *classSpecifier = simpleDecl->decl_specifier_list->value->asClassSpecifier();
+    QVERIFY(classSpecifier);
+
+    QVERIFY(diag.errorCount != 0);
+}
+
 void tst_AST::initTestCase()
 {
     control.setDiagnosticClient(&diag);