C++: Fix outdated macro usage info in documents.

Record revisions of documents in macro definitions and usages. Then,
when searching for usages, check the revision of the documents against
the revision of the macros. If they are out-of-sync, repreprocess the
documents to get up-to-date info.

Task-number: QTCREATORBUG-7872
Change-Id: I846bb52ec660024728ab117a9fb7e43382a50e63
Reviewed-by: Christian Stenger <christian.stenger@digia.com>

src/libs/cplusplus/CppDocument.cpp

@@ -676,10 +676,19 @@ void Snapshot::insert(Document::Ptr doc)
         _documents.insert(doc->fileName(), doc);
 }
 
-QByteArray Snapshot::preprocessedCode(const QString &source, const QString &fileName) const
+Document::Ptr Snapshot::preprocessedDocument(const QString &source, const QString &fileName) const
 {
+    Document::Ptr newDoc = Document::create(fileName);
+    if (Document::Ptr thisDocument = document(fileName)) {
+        newDoc->_revision = thisDocument->_revision;
+        newDoc->_editorRevision = thisDocument->_editorRevision;
+        newDoc->_lastModified = thisDocument->_lastModified;
+    }
+
     FastPreprocessor pp(*this);
-    return pp.run(fileName, source);
+    const QByteArray preprocessedCode = pp.run(newDoc, source);
+    newDoc->setUtf8Source(preprocessedCode);
+    return newDoc;
 }
 
 Document::Ptr Snapshot::documentFromSource(const QByteArray &preprocessedCode,

src/libs/cplusplus/CppDocument.h

@@ -79,7 +79,8 @@ public:
 
     void appendMacro(const Macro &macro);
     void addMacroUse(const Macro &macro, unsigned offset, unsigned length,
-                     unsigned beginLine, const QVector<MacroArgumentReference> &range);
+                     unsigned beginLine,
+                     const QVector<MacroArgumentReference> &range);
     void addUndefinedMacroUse(const QByteArray &name, unsigned offset);
 
     Control *control() const;
@@ -254,8 +255,7 @@ public:
         unsigned _beginLine;
 
     public:
-        inline MacroUse(const Macro &macro,
+        inline MacroUse(const Macro &macro, unsigned begin, unsigned end, unsigned beginLine)
-                        unsigned begin, unsigned end, unsigned beginLine)
             : Block(begin, end),
               _macro(macro),
               _beginLine(beginLine)
@@ -371,10 +371,10 @@ public:
 
     Snapshot simplified(Document::Ptr doc) const;
 
-    QByteArray preprocessedCode(const QString &source,
+    Document::Ptr preprocessedDocument(const QString &source,
                                        const QString &fileName) const;
 
-    Document::Ptr documentFromSource(const QByteArray &preprocessedCode,
+    Document::Ptr documentFromSource(const QByteArray &preprocessedDocument,
                                      const QString &fileName) const;
 
 private:

src/libs/cplusplus/FastPreprocessor.cpp

@@ -39,8 +39,10 @@ FastPreprocessor::FastPreprocessor(const Snapshot &snapshot)
       _preproc(this, &_env)
 { }
 
-QByteArray FastPreprocessor::run(QString fileName, const QString &source)
+QByteArray FastPreprocessor::run(Document::Ptr newDoc, const QString &source)
 {
+    std::swap(newDoc, _currentDoc);
+    const QString fileName = _currentDoc->fileName();
     _preproc.setExpandFunctionlikeMacros(false);
     _preproc.setKeepComments(true);
 
@@ -54,11 +56,17 @@ QByteArray FastPreprocessor::run(QString fileName, const QString &source)
 
     const QByteArray preprocessed = _preproc.run(fileName, source);
 //    qDebug("FastPreprocessor::run for %s produced [[%s]]", fileName.toUtf8().constData(), preprocessed.constData());
+    std::swap(newDoc, _currentDoc);
     return preprocessed;
 }
 
-void FastPreprocessor::sourceNeeded(unsigned, QString &fileName, IncludeType)
+void FastPreprocessor::sourceNeeded(unsigned line, QString &fileName, IncludeType)
-{ mergeEnvironment(fileName); }
+{
+    Q_ASSERT(_currentDoc);
+    _currentDoc->addIncludeFile(fileName, line);
+
+    mergeEnvironment(fileName);
+}
 
 void FastPreprocessor::mergeEnvironment(const QString &fileName)
 {
@@ -73,3 +81,56 @@ void FastPreprocessor::mergeEnvironment(const QString &fileName)
         }
     }
 }
+
+void FastPreprocessor::macroAdded(const Macro &macro)
+{
+    Q_ASSERT(_currentDoc);
+
+    _currentDoc->appendMacro(macro);
+}
+
+static const Macro revision(const Snapshot &s, const Macro &m)
+{
+    if (Document::Ptr d = s.document(m.fileName())) {
+        Macro newMacro(m);
+        newMacro.setFileRevision(d->revision());
+        return newMacro;
+    }
+
+    return m;
+}
+
+void FastPreprocessor::passedMacroDefinitionCheck(unsigned offset, unsigned line, const Macro &macro)
+{
+    Q_ASSERT(_currentDoc);
+
+    _currentDoc->addMacroUse(revision(_snapshot, macro),
+                             offset, macro.name().length(), line,
+                             QVector<MacroArgumentReference>());
+}
+
+void FastPreprocessor::failedMacroDefinitionCheck(unsigned offset, const ByteArrayRef &name)
+{
+    Q_ASSERT(_currentDoc);
+
+    _currentDoc->addUndefinedMacroUse(QByteArray(name.start(), name.size()), offset);
+}
+
+void FastPreprocessor::notifyMacroReference(unsigned offset, unsigned line, const Macro &macro)
+{
+    Q_ASSERT(_currentDoc);
+
+    _currentDoc->addMacroUse(revision(_snapshot, macro),
+                             offset, macro.name().length(), line,
+                             QVector<MacroArgumentReference>());
+}
+
+void FastPreprocessor::startExpandingMacro(unsigned offset, unsigned line,
+                                          const Macro &macro,
+                                          const QVector<MacroArgumentReference> &actuals)
+{
+    Q_ASSERT(_currentDoc);
+
+    _currentDoc->addMacroUse(revision(_snapshot, macro),
+                             offset, macro.name().length(), line, actuals);
+}

src/libs/cplusplus/FastPreprocessor.h

@@ -47,28 +47,29 @@ class CPLUSPLUS_EXPORT FastPreprocessor: public Client
     Snapshot _snapshot;
     Preprocessor _preproc;
     QSet<QString> _merged;
+    Document::Ptr _currentDoc;
 
     void mergeEnvironment(const QString &fileName);
 
 public:
     FastPreprocessor(const Snapshot &snapshot);
 
-    QByteArray run(QString fileName, const QString &source);
+    QByteArray run(Document::Ptr newDoc, const QString &source);
 
     // CPlusPlus::Client
-    virtual void sourceNeeded(unsigned, QString &fileName, IncludeType);
+    virtual void sourceNeeded(unsigned line, QString &fileName, IncludeType);
 
-    virtual void macroAdded(const Macro &) {}
+    virtual void macroAdded(const Macro &);
 
-    virtual void passedMacroDefinitionCheck(unsigned, unsigned, const Macro &) {}
+    virtual void passedMacroDefinitionCheck(unsigned, unsigned, const Macro &);
-    virtual void failedMacroDefinitionCheck(unsigned, const ByteArrayRef &) {}
+    virtual void failedMacroDefinitionCheck(unsigned, const ByteArrayRef &);
 
-    virtual void notifyMacroReference(unsigned, unsigned, const Macro &) {}
+    virtual void notifyMacroReference(unsigned, unsigned, const Macro &);
 
     virtual void startExpandingMacro(unsigned,
                                      unsigned,
                                      const Macro &,
-                                     const QVector<MacroArgumentReference> &) {}
+                                     const QVector<MacroArgumentReference> &);
     virtual void stopExpandingMacro(unsigned, const Macro &) {}
 
     virtual void startSkippingBlocks(unsigned) {}

src/libs/cplusplus/Macro.cpp

@@ -53,6 +53,7 @@ using namespace CPlusPlus;
 Macro::Macro()
     : _next(0),
       _hashcode(0),
+      _fileRevision(0),
       _line(0),
       _offset(0),
       _length(0),

src/libs/cplusplus/Macro.h

@@ -95,6 +95,12 @@ public:
     void setFileName(const QString &fileName)
     { _fileName = fileName; }
 
+    unsigned fileRevision() const
+    { return _fileRevision; }
+
+    void setFileRevision(unsigned fileRevision)
+    { _fileRevision = fileRevision; }
+
     unsigned line() const
     { return _line; }
 
@@ -154,6 +160,7 @@ private:
     QVector<PPToken> _definitionTokens;
     QVector<QByteArray> _formals;
     QString _fileName;
+    unsigned _fileRevision;
     unsigned _line;
     unsigned _offset;
     unsigned _length;

src/libs/cplusplus/pp-engine.cpp

@@ -812,6 +812,9 @@ bool Preprocessor::handleIdentifier(PPToken *tk)
         if (!expandFunctionlikeMacros()
                 // Still expand if this originally started with an object-like macro.
                 && m_state.m_expansionStatus != Expanding) {
+            m_client->notifyMacroReference(m_state.m_offsetRef + idTk.offset,
+                                           idTk.lineno,
+                                           *macro);
             return false;
         }
 

src/plugins/cppeditor/cppeditor.cpp

@@ -2140,10 +2140,7 @@ SemanticInfo SemanticHighlighter::semanticInfo(const Source &source)
     if (! semanticInfo.doc) {
         semanticInfo.snapshot = source.snapshot;
         if (source.snapshot.contains(source.fileName)) {
-            const QByteArray &preprocessedCode =
+            Document::Ptr doc = source.snapshot.preprocessedDocument(source.code, source.fileName);
-                    source.snapshot.preprocessedCode(source.code, source.fileName);
-            Document::Ptr doc =
-                    source.snapshot.documentFromSource(preprocessedCode, source.fileName);
             doc->control()->setTopLevelDeclarationProcessor(this);
             doc->check();
             semanticInfo.doc = doc;

src/plugins/cpptools/cppfindreferences.cpp

@@ -128,14 +128,12 @@ public:
                 return usages; // skip this document, it's not using symbolId.
         }
         Document::Ptr doc;
-        QByteArray source;
         const QString unpreprocessedSource = getSource(fileName, workingCopy);
 
         if (symbolDocument && fileName == symbolDocument->fileName()) {
             doc = symbolDocument;
         } else {
-            source = snapshot.preprocessedCode(unpreprocessedSource, fileName);
+            doc = snapshot.preprocessedDocument(unpreprocessedSource, fileName);
-            doc = snapshot.documentFromSource(source, fileName);
             doc->tokenize();
         }
 
@@ -458,10 +456,8 @@ bool CppFindReferences::findSymbol(CppFindReferencesParameters *parameters,
     Document::Ptr newSymbolDocument = snapshot.document(symbolFile);
     // document is not parsed and has no bindings yet, do it
     QString source = getSource(newSymbolDocument->fileName(), _modelManager->workingCopy());
-    const QByteArray &preprocessedCode =
-            snapshot.preprocessedCode(source, newSymbolDocument->fileName());
     Document::Ptr doc =
-            snapshot.documentFromSource(preprocessedCode, newSymbolDocument->fileName());
+            snapshot.preprocessedDocument(source, newSymbolDocument->fileName());
     doc->check();
 
     // construct id of old symbol
@@ -563,19 +559,29 @@ public:
     QList<Usage> operator()(const QString &fileName)
     {
         QList<Usage> usages;
+        Document::Ptr doc = snapshot.document(fileName);
+        QString source;
+
+_Lrestart:
         if (future->isPaused())
             future->waitForResume();
         if (future->isCanceled())
             return usages;
 
-        const Document::Ptr &doc = snapshot.document(fileName);
+        usages.clear();
-        QString source;
-
         foreach (const Document::MacroUse &use, doc->macroUses()) {
             const Macro &useMacro = use.macro();
-            if (useMacro.line() == macro.line()
+
-                && useMacro.fileName() == macro.fileName())
+            if (useMacro.fileName() == macro.fileName()) { // Check if this is a match, but possibly against an outdated document.
-                {
+                if (macro.fileRevision() > useMacro.fileRevision()) {
+                    // yes, it is outdated, so re-preprocess and start from scratch for this file.
+                    source = getSource(fileName, workingCopy).toLatin1();
+                    doc = snapshot.preprocessedDocument(source, fileName);
+                    goto _Lrestart;
+                }
+            }
+
+            if (useMacro.fileName() == macro.fileName() && macro.name() == useMacro.name()) {
                 if (source.isEmpty())
                     source = getSource(fileName, workingCopy);
 
@@ -625,7 +631,6 @@ static void findMacroUses_helper(QFutureInterface<Usage> &future,
     files.removeDuplicates();
 
     future.setProgressRange(0, files.size());
-
     FindMacroUsesInFile process(workingCopy, snapshot, macro, &future);
     UpdateUI reduce(&future);
     // This thread waits for blockingMappedReduced to finish, so reduce the pool's used thread count

src/plugins/cpptools/cppmodelmanager.cpp

@@ -498,12 +498,19 @@ void CppPreprocessor::macroAdded(const Macro &macro)
     m_currentDoc->appendMacro(macro);
 }
 
+static inline const Macro revision(const CppModelManagerInterface::WorkingCopy &s, const Macro &macro)
+{
+    Macro newMacro(macro);
+    newMacro.setFileRevision(s.get(macro.fileName()).second);
+    return newMacro;
+}
+
 void CppPreprocessor::passedMacroDefinitionCheck(unsigned offset, unsigned line, const Macro &macro)
 {
     if (! m_currentDoc)
         return;
 
-    m_currentDoc->addMacroUse(macro, offset, macro.name().length(), line,
+    m_currentDoc->addMacroUse(revision(m_workingCopy, macro), offset, macro.name().length(), line,
                               QVector<MacroArgumentReference>());
 }
 
@@ -520,7 +527,7 @@ void CppPreprocessor::notifyMacroReference(unsigned offset, unsigned line, const
     if (! m_currentDoc)
         return;
 
-    m_currentDoc->addMacroUse(macro, offset, macro.name().length(), line,
+    m_currentDoc->addMacroUse(revision(m_workingCopy, macro), offset, macro.name().length(), line,
                               QVector<MacroArgumentReference>());
 }
 
@@ -531,7 +538,7 @@ void CppPreprocessor::startExpandingMacro(unsigned offset, unsigned line,
     if (! m_currentDoc)
         return;
 
-    m_currentDoc->addMacroUse(macro, offset, macro.name().length(), line, actuals);
+    m_currentDoc->addMacroUse(revision(m_workingCopy, macro), offset, macro.name().length(), line, actuals);
 }
 
 void CppPreprocessor::stopExpandingMacro(unsigned, const Macro &)

src/plugins/cpptools/cpprefactoringchanges.cpp

@@ -156,8 +156,7 @@ Document::Ptr CppRefactoringFile::cppDocument() const
         const QString name = fileName();
         const Snapshot &snapshot = data()->m_snapshot;
 
-        const QByteArray contents = snapshot.preprocessedCode(source, name);
+        m_cppDocument = snapshot.preprocessedDocument(source, name);
-        m_cppDocument = snapshot.documentFromSource(contents, name);
         m_cppDocument->check();
     }
 

src/plugins/debugger/cdb/cdbengine.cpp

@@ -2626,9 +2626,7 @@ static CPlusPlus::Document::Ptr getParsedDocument(const QString &fileName,
             src = QString::fromLocal8Bit(reader.data()); // ### FIXME encoding
     }
 
-    QByteArray source = snapshot.preprocessedCode(src, fileName);
+    CPlusPlus::Document::Ptr doc = snapshot.preprocessedDocument(src, fileName);
-
-    CPlusPlus::Document::Ptr doc = snapshot.documentFromSource(source, fileName);
     doc->parse();
     return doc;
 }

src/plugins/designer/qtcreatorintegration.cpp

@@ -511,9 +511,7 @@ static Document::Ptr getParsedDocument(const QString &fileName, CppModelManagerI
             src = QString::fromLocal8Bit(reader.data()); // ### FIXME encoding
     }
 
-    QByteArray source = snapshot.preprocessedCode(src, fileName);
+    Document::Ptr doc = snapshot.preprocessedDocument(src, fileName);
-
-    Document::Ptr doc = snapshot.documentFromSource(source, fileName);
     doc->check();
     snapshot.insert(doc);
     return doc;