Clang: Refactor TokenInfos

Remove code duplication.

Change-Id: Ib1859f2c3a04f66d0f0b669b4e93a7fc06ab8e61
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@qt.io>

src/plugins/clangcodemodel/clangcodemodel.pro

@@ -31,8 +31,8 @@ SOURCES += \
     clangfixitoperationsextractor.cpp \
     clangfollowsymbol.cpp \
     clangfunctionhintmodel.cpp \
+    clanghighlightingresultreporter.cpp \
     clanghoverhandler.cpp \
-    clangtokeninfosreporter.cpp \
     clangmodelmanagersupport.cpp \
     clangpreprocessorassistproposalitem.cpp \
     clangprojectsettings.cpp \
@@ -69,6 +69,7 @@ HEADERS += \
     clangfixitoperationsextractor.h \
     clangfollowsymbol.h \
     clangfunctionhintmodel.h \
+    clanghighlightingresultreporter.h \
     clanghoverhandler.h \
     clangisdiagnosticrelatedtolocation.h \
     clangmodelmanagersupport.h \
@@ -77,7 +78,6 @@ HEADERS += \
     clangprojectsettingswidget.h \
     clangrefactoringengine.h \
     clangtextmark.h \
-    clangtokeninfosreporter.h \
     clanguiheaderondiskmanager.h \
     clangutils.h
 

src/plugins/clangcodemodel/clangcodemodel.qbs

@@ -82,10 +82,10 @@ QtcPlugin {
         "clangfollowsymbol.h",
         "clangfunctionhintmodel.cpp",
         "clangfunctionhintmodel.h",
+        "clanghighlightingresultreporter.cpp",
+        "clanghighlightingresultreporter.h",
         "clanghoverhandler.cpp",
         "clanghoverhandler.h",
-        "clangtokeninfosreporter.cpp",
-        "clangtokeninfosreporter.h",
         "clangisdiagnosticrelatedtolocation.h",
         "clangmodelmanagersupport.cpp",
         "clangmodelmanagersupport.h",

src/plugins/clangcodemodel/clangcodemodelunittestfiles.pri

@@ -7,7 +7,7 @@ SOURCES += \
     $$PWD/clangcompletioncontextanalyzer.cpp \
     $$PWD/clangdiagnosticfilter.cpp \
     $$PWD/clangfixitoperation.cpp \
-    $$PWD/clangtokeninfosreporter.cpp
+    $$PWD/clanghighlightingresultreporter.cpp
 
 HEADERS += \
     $$PWD/clangactivationsequencecontextprocessor.h \
@@ -16,5 +16,5 @@ HEADERS += \
     $$PWD/clangcompletioncontextanalyzer.h \
     $$PWD/clangdiagnosticfilter.h \
     $$PWD/clangfixitoperation.h \
-    $$PWD/clangtokeninfosreporter.h \
+    $$PWD/clanghighlightingresultreporter.h \
     $$PWD/clangisdiagnosticrelatedtolocation.h

src/plugins/clangcodemodel/clangeditordocumentprocessor.cpp

@@ -30,7 +30,7 @@
 #include "clangfixitoperation.h"
 #include "clangfixitoperationsextractor.h"
 #include "clangmodelmanagersupport.h"
-#include "clangtokeninfosreporter.h"
+#include "clanghighlightingresultreporter.h"
 #include "clangprojectsettings.h"
 #include "clangutils.h"
 
@@ -253,7 +253,7 @@ void ClangEditorDocumentProcessor::updateHighlighting(
         m_tokenInfos = tokenInfos;
         m_semanticHighlighter.setHighlightingRunner(
             [tokenInfos]() {
-                auto *reporter = new TokenInfosReporter(tokenInfos);
+                auto *reporter = new HighlightingResultReporter(tokenInfos);
                 return reporter->start();
             });
         m_semanticHighlighter.run();

src/plugins/clangcodemodel/clanghighlightingresultreporter.cpp

@@ -23,7 +23,7 @@
 **
 ****************************************************************************/
 
-#include "clangtokeninfosreporter.h"
+#include "clanghighlightingresultreporter.h"
 
 #include <texteditor/textstyles.h>
 #include <utils/qtcassert.h>
@@ -134,14 +134,14 @@ TextEditor::HighlightingResult toHighlightingResult(
 
 namespace ClangCodeModel {
 
-TokenInfosReporter::TokenInfosReporter(
+HighlightingResultReporter::HighlightingResultReporter(
         const QVector<ClangBackEnd::TokenInfoContainer> &tokenInfos)
     : m_tokenInfos(tokenInfos)
 {
     m_chunksToReport.reserve(m_chunkSize + 1);
 }
 
-void TokenInfosReporter::reportChunkWise(
+void HighlightingResultReporter::reportChunkWise(
         const TextEditor::HighlightingResult &highlightingResult)
 {
     if (m_chunksToReport.size() >= m_chunkSize) {
@@ -156,7 +156,7 @@ void TokenInfosReporter::reportChunkWise(
     m_chunksToReport.append(highlightingResult);
 }
 
-void TokenInfosReporter::reportAndClearCurrentChunks()
+void HighlightingResultReporter::reportAndClearCurrentChunks()
 {
     m_flushRequested = false;
     m_flushLine = 0;
@@ -167,18 +167,18 @@ void TokenInfosReporter::reportAndClearCurrentChunks()
     }
 }
 
-void TokenInfosReporter::setChunkSize(int chunkSize)
+void HighlightingResultReporter::setChunkSize(int chunkSize)
 {
     m_chunkSize = chunkSize;
 }
 
-void TokenInfosReporter::run()
+void HighlightingResultReporter::run()
 {
     run_internal();
     reportFinished();
 }
 
-void TokenInfosReporter::run_internal()
+void HighlightingResultReporter::run_internal()
 {
     if (isCanceled())
         return;
@@ -199,7 +199,7 @@ void TokenInfosReporter::run_internal()
     reportAndClearCurrentChunks();
 }
 
-QFuture<TextEditor::HighlightingResult> TokenInfosReporter::start()
+QFuture<TextEditor::HighlightingResult> HighlightingResultReporter::start()
 {
     this->setRunnable(this);
     this->reportStarted();

src/plugins/clangcodemodel/clanghighlightingresultreporter.h

@@ -36,7 +36,7 @@
 
 namespace ClangCodeModel {
 
-class TokenInfosReporter:
+class HighlightingResultReporter:
         public QObject,
         public QRunnable,
         public QFutureInterface<TextEditor::HighlightingResult>
@@ -44,7 +44,7 @@ class TokenInfosReporter:
     Q_OBJECT
 
 public:
-    TokenInfosReporter(const QVector<ClangBackEnd::TokenInfoContainer> &tokenInfos);
+    HighlightingResultReporter(const QVector<ClangBackEnd::TokenInfoContainer> &tokenInfos);
 
     void setChunkSize(int chunkSize);
 

src/tools/clangbackend/source/clangbackendclangipc-source.pri

@@ -52,15 +52,14 @@ HEADERS += \
     $$PWD/diagnosticsetiterator.h \
     $$PWD/fixit.h \
     $$PWD/fulltokeninfo.h \
-    $$PWD/fulltokeninfos.h \
     $$PWD/projectpart.h \
     $$PWD/projects.h \
     $$PWD/skippedsourceranges.h \
     $$PWD/sourcelocation.h \
     $$PWD/sourcerange.h \
     $$PWD/tokeninfo.h \
-    $$PWD/tokeninfos.h \
-    $$PWD/tokeninfositerator.h \
+    $$PWD/tokenprocessor.h \
+    $$PWD/tokenprocessoriterator.h \
     $$PWD/unsavedfile.h \
     $$PWD/unsavedfiles.h \
     $$PWD/utf8positionfromlinecolumn.h
@@ -111,14 +110,12 @@ SOURCES += \
     $$PWD/diagnosticset.cpp \
     $$PWD/fixit.cpp \
     $$PWD/fulltokeninfo.cpp \
-    $$PWD/fulltokeninfos.cpp \
     $$PWD/projectpart.cpp \
     $$PWD/projects.cpp \
     $$PWD/skippedsourceranges.cpp \
     $$PWD/sourcelocation.cpp \
     $$PWD/sourcerange.cpp \
     $$PWD/tokeninfo.cpp \
-    $$PWD/tokeninfos.cpp \
     $$PWD/unsavedfile.cpp \
     $$PWD/unsavedfiles.cpp \
     $$PWD/utf8positionfromlinecolumn.cpp

src/tools/clangbackend/source/clangcodemodelserver.cpp

@@ -30,7 +30,7 @@
 #include "clangfilesystemwatcher.h"
 #include "codecompleter.h"
 #include "diagnosticset.h"
-#include "tokeninfos.h"
+#include "tokenprocessor.h"
 #include "clangexceptions.h"
 #include "skippedsourceranges.h"
 

src/tools/clangbackend/source/clangdocuments.cpp

@@ -26,7 +26,7 @@
 #include "clangdocuments.h"
 
 #include <diagnosticset.h>
-#include <tokeninfos.h>
+#include <tokenprocessor.h>
 #include <clangexceptions.h>
 #include <projects.h>
 #include <skippedsourceranges.h>

src/tools/clangbackend/source/clangtranslationunit.cpp

@@ -31,13 +31,12 @@
 #include "clangtranslationunitupdater.h"
 #include "clangfollowsymbol.h"
 #include "clangfollowsymboljob.h"
+#include "tokenprocessor.h"
 
 #include <codecompleter.h>
 #include <cursor.h>
 #include <diagnosticcontainer.h>
 #include <diagnosticset.h>
-#include <tokeninfos.h>
-#include <fulltokeninfos.h>
 #include <skippedsourceranges.h>
 #include <sourcelocation.h>
 #include <sourcerange.h>
@@ -201,34 +200,24 @@ Cursor TranslationUnit::cursor() const
     return clang_getTranslationUnitCursor(m_cxTranslationUnit);
 }
 
-TokenInfos TranslationUnit::tokenInfos() const
+TokenProcessor<TokenInfo> TranslationUnit::tokenInfos() const
 {
     return tokenInfosInRange(cursor().sourceRange());
 }
 
-TokenInfos TranslationUnit::tokenInfosInRange(const SourceRange &range) const
+TokenProcessor<TokenInfo> TranslationUnit::tokenInfosInRange(const SourceRange &range) const
 {
-    CXToken *cxTokens = 0;
-    uint cxTokensCount = 0;
-
-    clang_tokenize(m_cxTranslationUnit, range, &cxTokens, &cxTokensCount);
-
-    return TokenInfos(m_cxTranslationUnit, cxTokens, cxTokensCount);
+    return TokenProcessor<TokenInfo>(m_cxTranslationUnit, range);
 }
 
-FullTokenInfos TranslationUnit::fullTokenInfos() const
+TokenProcessor<FullTokenInfo> TranslationUnit::fullTokenInfos() const
 {
     return fullTokenInfosInRange(cursor().sourceRange());
 }
 
-FullTokenInfos TranslationUnit::fullTokenInfosInRange(const SourceRange &range) const
+TokenProcessor<FullTokenInfo> TranslationUnit::fullTokenInfosInRange(const SourceRange &range) const
 {
-    CXToken *cxTokens = 0;
-    uint cxTokensCount = 0;
-
-    clang_tokenize(m_cxTranslationUnit, range, &cxTokens, &cxTokensCount);
-
-    return FullTokenInfos(m_cxTranslationUnit, cxTokens, cxTokensCount);
+    return TokenProcessor<FullTokenInfo>(m_cxTranslationUnit, range);
 }
 
 SkippedSourceRanges TranslationUnit::skippedSourceRanges() const

src/tools/clangbackend/source/clangtranslationunit.h

@@ -25,6 +25,9 @@
 
 #pragma once
 
+#include "fulltokeninfo.h"
+#include "tokenprocessor.h"
+
 #include <clangsupport/codecompletion.h>
 
 #include <clang-c/Index.h>
@@ -34,9 +37,6 @@ namespace ClangBackEnd {
 class Cursor;
 class DiagnosticContainer;
 class DiagnosticSet;
-class TokenInfoContainer;
-class TokenInfos;
-class FullTokenInfos;
 class ReferencesResult;
 class SkippedSourceRanges;
 class SourceLocation;
@@ -101,11 +101,11 @@ public:
     Cursor cursorAt(const Utf8String &filePath, uint line, uint column) const;
     Cursor cursor() const;
 
-    TokenInfos tokenInfos() const;
-    TokenInfos tokenInfosInRange(const SourceRange &range) const;
+    TokenProcessor<TokenInfo> tokenInfos() const;
+    TokenProcessor<TokenInfo> tokenInfosInRange(const SourceRange &range) const;
 
-    FullTokenInfos fullTokenInfos() const;
-    FullTokenInfos fullTokenInfosInRange(const SourceRange &range) const;
+    TokenProcessor<FullTokenInfo> fullTokenInfos() const;
+    TokenProcessor<FullTokenInfo> fullTokenInfosInRange(const SourceRange &range) const;
 
     SkippedSourceRanges skippedSourceRanges() const;
     SourceRangeContainer followSymbol(uint line, uint column) const;

src/tools/clangbackend/source/clangupdateextradocumentannotationsjob.cpp

@@ -24,7 +24,6 @@
 ****************************************************************************/
 
 #include "clangupdateextradocumentannotationsjob.h"
-#include "fulltokeninfos.h"
 
 #include <clangsupport/clangsupportdebugutils.h>
 #include <clangsupport/clangcodemodelclientinterface.h>

src/tools/clangbackend/source/fulltokeninfos.cpp

@@ -1,93 +0,0 @@
-/****************************************************************************
-**
-** Copyright (C) 2018 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of Qt Creator.
-**
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 3 as published by the Free Software
-** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-3.0.html.
-**
-****************************************************************************/
-
-#include "fulltokeninfos.h"
-
-#include <clangsupport/tokeninfocontainer.h>
-
-#include <QVector>
-
-namespace ClangBackEnd {
-
-FullTokenInfos::FullTokenInfos(CXTranslationUnit cxTranslationUnit, CXToken *tokens, uint tokensCount)
-    : cxTranslationUnit(cxTranslationUnit),
-      cxTokens(tokens),
-      cxTokenCount(tokensCount)
-{
-    cxCursors.resize(tokensCount);
-    clang_annotateTokens(cxTranslationUnit, cxTokens, cxTokenCount, cxCursors.data());
-}
-
-FullTokenInfos::~FullTokenInfos()
-{
-    clang_disposeTokens(cxTranslationUnit, cxTokens, cxTokenCount);
-}
-
-QVector<TokenInfoContainer> FullTokenInfos::toTokenInfoContainers() const
-{
-    QVector<TokenInfoContainer> containers;
-    containers.reserve(static_cast<int>(size()));
-
-    const auto isValidTokenInfo = [] (const TokenInfo &tokenInfo) {
-        // Do not exclude StringLiteral because it can be a filename for an #include
-        return !tokenInfo.hasInvalidMainType()
-                && !tokenInfo.hasMainType(HighlightingType::NumberLiteral)
-                && !tokenInfo.hasMainType(HighlightingType::Comment);
-    };
-    for (size_t index = 0; index < cxCursors.size(); ++index) {
-        FullTokenInfo fullTokenInfo = (*this)[index];
-        if (isValidTokenInfo(fullTokenInfo))
-            containers.push_back(fullTokenInfo);
-    }
-
-    return containers;
-}
-
-bool FullTokenInfos::isEmpty() const
-{
-    return cxTokenCount == 0;
-}
-
-bool FullTokenInfos::isNull() const
-{
-    return cxTokens == nullptr;
-}
-
-size_t FullTokenInfos::size() const
-{
-    return cxTokenCount;
-}
-
-FullTokenInfo FullTokenInfos::operator[](size_t index) const
-{
-    FullTokenInfo tokenInfo(cxCursors[index],
-                            cxTokens + index,
-                            cxTranslationUnit,
-                            currentOutputArgumentRanges);
-    tokenInfo.evaluate();
-    return tokenInfo;
-}
-
-} // namespace ClangBackEnd

src/tools/clangbackend/source/fulltokeninfos.h

@@ -1,59 +0,0 @@
-/****************************************************************************
-**
-** Copyright (C) 2018 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of Qt Creator.
-**
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 3 as published by the Free Software
-** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-3.0.html.
-**
-****************************************************************************/
-
-#pragma once
-
-#include "fulltokeninfo.h"
-
-#include <clang-c/Index.h>
-
-#include <vector>
-
-namespace ClangBackEnd {
-
-class FullTokenInfos
-{
-public:
-    FullTokenInfos() = default;
-    FullTokenInfos(CXTranslationUnit cxTranslationUnit, CXToken *tokens, uint tokensCount);
-    ~FullTokenInfos();
-
-    bool isEmpty() const;
-    bool isNull() const;
-    size_t size() const;
-
-    FullTokenInfo operator[](size_t index) const;
-    QVector<TokenInfoContainer> toTokenInfoContainers() const;
-
-private:
-    mutable std::vector<CXSourceRange> currentOutputArgumentRanges;
-    CXTranslationUnit cxTranslationUnit = nullptr;
-    CXToken *const cxTokens = nullptr;
-    const uint cxTokenCount = 0;
-
-    std::vector<CXCursor> cxCursors;
-};
-
-} // namespace ClangBackEnd

src/tools/clangbackend/source/tokeninfos.cpp

@@ -1,111 +0,0 @@
-/****************************************************************************
-**
-** Copyright (C) 2016 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of Qt Creator.
-**
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 3 as published by the Free Software
-** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-3.0.html.
-**
-****************************************************************************/
-
-#include "tokeninfos.h"
-
-#include "tokeninfocontainer.h"
-
-#include <QVector>
-
-namespace ClangBackEnd {
-
-TokenInfos::TokenInfos(CXTranslationUnit cxTranslationUnit, CXToken *tokens, uint tokensCount)
-    : cxTranslationUnit(cxTranslationUnit),
-      cxTokens(tokens),
-      cxTokenCount(tokensCount)
-{
-    cxCursors.resize(tokensCount);
-    clang_annotateTokens(cxTranslationUnit, cxTokens, cxTokenCount, cxCursors.data());
-}
-
-TokenInfos::~TokenInfos()
-{
-    clang_disposeTokens(cxTranslationUnit, cxTokens, cxTokenCount);
-}
-
-TokenInfos::const_iterator TokenInfos::begin() const
-{
-    return const_iterator(cxCursors.cbegin(),
-                          cxTokens,
-                          cxTranslationUnit,
-                          currentOutputArgumentRanges);
-}
-
-TokenInfos::const_iterator TokenInfos::end() const
-{
-    return const_iterator(cxCursors.cend(),
-                          cxTokens + cxTokenCount,
-                          cxTranslationUnit,
-                          currentOutputArgumentRanges);
-}
-
-QVector<TokenInfoContainer> TokenInfos::toTokenInfoContainers() const
-{
-    QVector<TokenInfoContainer> containers;
-    containers.reserve(size());
-
-    const auto isValidTokenInfo = [] (const TokenInfo &tokenInfo) {
-        return !tokenInfo.hasInvalidMainType()
-                && !tokenInfo.hasMainType(HighlightingType::NumberLiteral)
-                && !tokenInfo.hasMainType(HighlightingType::Comment);
-    };
-    for (const TokenInfo &tokenInfo : *this)
-        if (isValidTokenInfo(tokenInfo))
-            containers.push_back(tokenInfo);
-
-    return containers;
-}
-
-bool TokenInfos::currentOutputArgumentRangesAreEmpty() const
-{
-    return currentOutputArgumentRanges.empty();
-}
-
-bool TokenInfos::isEmpty() const
-{
-    return cxTokenCount == 0;
-}
-
-bool ClangBackEnd::TokenInfos::isNull() const
-{
-    return cxTokens == nullptr;
-}
-
-uint TokenInfos::size() const
-{
-    return cxTokenCount;
-}
-
-TokenInfo TokenInfos::operator[](size_t index) const
-{
-    TokenInfo tokenInfo(cxCursors[index],
-                        cxTokens + index,
-                        cxTranslationUnit,
-                        currentOutputArgumentRanges);
-    tokenInfo.evaluate();
-    return tokenInfo;
-}
-
-} // namespace ClangBackEnd

src/tools/clangbackend/source/tokeninfos.h

@@ -1,72 +0,0 @@
-/****************************************************************************
-**
-** Copyright (C) 2016 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of Qt Creator.
-**
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 3 as published by the Free Software
-** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-3.0.html.
-**
-****************************************************************************/
-
-#pragma once
-
-#include "tokeninfositerator.h"
-
-#include <clang-c/Index.h>
-
-#include <vector>
-
-namespace ClangBackEnd {
-
-using uint = unsigned int;
-class TokenInfoContainer;
-
-class TokenInfos
-{
-public:
-    using const_iterator = TokenInfosIterator;
-    using value_type = TokenInfo;
-
-public:
-    TokenInfos() = default;
-    TokenInfos(CXTranslationUnit cxTranslationUnit, CXToken *tokens, uint tokensCount);
-    ~TokenInfos();
-
-    bool isEmpty() const;
-    bool isNull() const;
-    uint size() const;
-
-    TokenInfo operator[](size_t index) const;
-
-    const_iterator begin() const;
-    const_iterator end() const;
-
-    QVector<TokenInfoContainer> toTokenInfoContainers() const;
-
-    bool currentOutputArgumentRangesAreEmpty() const;
-
-private:
-    mutable std::vector<CXSourceRange> currentOutputArgumentRanges;
-    CXTranslationUnit cxTranslationUnit = nullptr;
-    CXToken *const cxTokens = nullptr;
-    const uint cxTokenCount = 0;
-
-    std::vector<CXCursor> cxCursors;
-};
-
-} // namespace ClangBackEnd

src/tools/clangbackend/source/tokenprocessor.h

@@ -0,0 +1,137 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "sourcerange.h"
+#include "tokenprocessoriterator.h"
+#include "tokeninfocontainer.h"
+
+#include <clang-c/Index.h>
+
+#include <QVector>
+
+#include <vector>
+
+namespace ClangBackEnd {
+
+using uint = unsigned int;
+
+template<class T>
+class TokenProcessor
+{
+    static_assert (std::is_base_of<TokenInfo, T>::value,
+                   "Use TokenProcessor only with classes derived from TokenInfo");
+
+public:
+    using const_iterator = TokenProcessorIterator<T>;
+    using value_type = T;
+
+public:
+    TokenProcessor() = default;
+    TokenProcessor(CXTranslationUnit cxTranslationUnit, const SourceRange &range)
+        : cxTranslationUnit(cxTranslationUnit)
+    {
+        uint cxTokensCount = 0;
+        clang_tokenize(cxTranslationUnit, range, &cxTokens, &cxTokensCount);
+        cxCursors.resize(cxTokensCount);
+        clang_annotateTokens(cxTranslationUnit, cxTokens, cxTokensCount, cxCursors.data());
+    }
+    ~TokenProcessor()
+    {
+        clang_disposeTokens(cxTranslationUnit, cxTokens, cxCursors.size());
+    }
+
+    bool isEmpty() const
+    {
+        return cxCursors.empty();
+    }
+    bool isNull() const
+    {
+        return cxTokens == nullptr;
+    }
+    uint size() const
+    {
+        return cxCursors.size();
+    }
+
+    const_iterator begin() const
+    {
+        return const_iterator(cxCursors.cbegin(),
+                              cxTokens,
+                              cxTranslationUnit,
+                              currentOutputArgumentRanges);
+    }
+
+    const_iterator end() const
+    {
+        return const_iterator(cxCursors.cend(),
+                              cxTokens + cxCursors.size(),
+                              cxTranslationUnit,
+                              currentOutputArgumentRanges);
+    }
+
+
+    T operator[](size_t index) const
+    {
+        T tokenInfo(cxCursors[index], cxTokens + index, cxTranslationUnit,
+                    currentOutputArgumentRanges);
+        tokenInfo.evaluate();
+        return tokenInfo;
+    }
+
+    QVector<TokenInfoContainer> toTokenInfoContainers() const
+    {
+        QVector<TokenInfoContainer> containers;
+        containers.reserve(size());
+
+        const auto isValidTokenInfo = [] (const T &tokenInfo) {
+            return !tokenInfo.hasInvalidMainType()
+                    && !tokenInfo.hasMainType(HighlightingType::NumberLiteral)
+                    && !tokenInfo.hasMainType(HighlightingType::Comment);
+        };
+        for (size_t index = 0; index < cxCursors.size(); ++index) {
+            T tokenInfo = (*this)[index];
+            if (isValidTokenInfo(tokenInfo))
+                containers.push_back(tokenInfo);
+        }
+
+        return containers;
+    }
+
+    bool currentOutputArgumentRangesAreEmpty() const
+    {
+        return currentOutputArgumentRanges.empty();
+    }
+
+private:
+    mutable std::vector<CXSourceRange> currentOutputArgumentRanges;
+    CXTranslationUnit cxTranslationUnit = nullptr;
+    CXToken *cxTokens = nullptr;
+
+    std::vector<CXCursor> cxCursors;
+};
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/tokenprocessoriterator.h

@@ -39,20 +39,21 @@ using uint = unsigned int;
 class DiagnosticSet;
 class Diagnostic;
 
-class TokenInfosIterator : public std::iterator<std::forward_iterator_tag, TokenInfo, uint>
+template<class T>
+class TokenProcessorIterator : public std::iterator<std::forward_iterator_tag, TokenInfo, uint>
 {
 public:
-    TokenInfosIterator(std::vector<CXCursor>::const_iterator cxCursorIterator,
-                              CXToken *cxToken,
-                              CXTranslationUnit cxTranslationUnit,
-                              std::vector<CXSourceRange> &currentOutputArgumentRanges)
+    TokenProcessorIterator(std::vector<CXCursor>::const_iterator cxCursorIterator,
+                           CXToken *cxToken,
+                           CXTranslationUnit cxTranslationUnit,
+                           std::vector<CXSourceRange> &currentOutputArgumentRanges)
         : cxCursorIterator(cxCursorIterator),
           cxToken(cxToken),
           cxTranslationUnit(cxTranslationUnit),
           currentOutputArgumentRanges(currentOutputArgumentRanges)
     {}
 
-    TokenInfosIterator& operator++()
+    TokenProcessorIterator& operator++()
     {
         ++cxCursorIterator;
         ++cxToken;
@@ -60,30 +61,27 @@ public:
         return *this;
     }
 
-    TokenInfosIterator operator++(int)
+    TokenProcessorIterator operator++(int)
     {
-        return TokenInfosIterator(cxCursorIterator++,
-                                  cxToken++,
-                                  cxTranslationUnit,
-                                  currentOutputArgumentRanges);
+        return TokenProcessorIterator(cxCursorIterator++,
+                                      cxToken++,
+                                      cxTranslationUnit,
+                                      currentOutputArgumentRanges);
     }
 
-    bool operator==(TokenInfosIterator other) const
+    bool operator==(TokenProcessorIterator other) const
     {
         return cxCursorIterator == other.cxCursorIterator;
     }
 
-    bool operator!=(TokenInfosIterator other) const
+    bool operator!=(TokenProcessorIterator other) const
     {
         return cxCursorIterator != other.cxCursorIterator;
     }
 
-    TokenInfo operator*()
+    T operator*()
     {
-        TokenInfo tokenInfo(*cxCursorIterator,
-                            cxToken,
-                            cxTranslationUnit,
-                            currentOutputArgumentRanges);
+        T tokenInfo(*cxCursorIterator, cxToken, cxTranslationUnit, currentOutputArgumentRanges);
         tokenInfo.evaluate();
         return tokenInfo;
     }

tests/unit/unittest/clangdocument-test.cpp

@@ -32,7 +32,7 @@
 #include <clangtranslationunit.h>
 #include <commandlinearguments.h>
 #include <diagnosticset.h>
-#include <tokeninfos.h>
+#include <tokenprocessor.h>
 #include <filecontainer.h>
 #include <projectpart.h>
 #include <projectpartcontainer.h>

tests/unit/unittest/gtest-creator-printing.cpp

@@ -37,12 +37,13 @@
 #include <clangrefactoringmessages.h>
 #include <filestatus.h>
 #include <filepath.h>
+#include <fulltokeninfo.h>
 #include <nativefilepath.h>
 #include <precompiledheadersupdatedmessage.h>
 #include <sourcedependency.h>
 #include <sourcelocationentry.h>
 #include <sourcelocationscontainer.h>
-#include <tokeninfos.h>
+#include <tokenprocessor.h>
 #include <filepathview.h>
 #include <tooltipinfo.h>
 #include <projectpartentry.h>
@@ -820,11 +821,12 @@ std::ostream &operator<<(std::ostream &os, const TokenInfo& tokenInfo)
     return  os;
 }
 
-std::ostream &operator<<(std::ostream &out, const TokenInfos &tokenInfos)
+template<class T>
+std::ostream &operator<<(std::ostream &out, const TokenProcessor<T> &tokenInfos)
 {
     out << "[";
 
-    for (const TokenInfo &entry : tokenInfos)
+    for (const T &entry : tokenInfos)
         out << entry;
 
     out << "]";
@@ -832,6 +834,11 @@ std::ostream &operator<<(std::ostream &out, const TokenInfos &tokenInfos)
     return out;
 }
 
+template
+std::ostream &operator<<(std::ostream &out, const TokenProcessor<TokenInfo> &tokenInfos);
+template
+std::ostream &operator<<(std::ostream &out, const TokenProcessor<FullTokenInfo> &tokenInfos);
+
 std::ostream &operator<<(std::ostream &out, const FilePath &filePath)
 {
     return out << "(" << filePath.path() << ", " << filePath.slashIndex() << ")";

tests/unit/unittest/gtest-creator-printing.h

@@ -98,6 +98,7 @@ class DynamicASTMatcherDiagnosticContextContainer;
 class DynamicASTMatcherDiagnosticMessageContainer;
 class FileContainer;
 class FixItContainer;
+class FullTokenInfo;
 class HighlightingMarkContainer;
 class NativeFilePath;
 class PrecompiledHeadersUpdatedMessage;
@@ -119,13 +120,14 @@ class SourceRangesAndDiagnosticsForQueryMessage;
 class SourceRangesContainer;
 class SourceRangesForQueryMessage;
 class SourceRangeWithTextContainer;
+class TokenInfo;
+template<class T>
+class TokenProcessor;
 class UnregisterUnsavedFilesForEditorMessage;
 class UpdatePchProjectPartsMessage;
 class UpdateTranslationUnitsForEditorMessage;
 class UpdateVisibleTranslationUnitsMessage;
 class FilePath;
-class TokenInfo;
-class TokenInfos;
 template <char WindowsSlash>
 class AbstractFilePathView;
 using FilePathView = AbstractFilePathView<'/'>;
@@ -193,7 +195,12 @@ std::ostream &operator<<(std::ostream &out, const UpdateVisibleTranslationUnitsM
 std::ostream &operator<<(std::ostream &out, const FilePath &filePath);
 std::ostream &operator<<(std::ostream &out, const FilePathId &filePathId);
 std::ostream &operator<<(std::ostream &out, const TokenInfo& tokenInfo);
-std::ostream &operator<<(std::ostream &out, const TokenInfos &tokenInfos);
+template<class T>
+std::ostream &operator<<(std::ostream &out, const TokenProcessor<T> &tokenInfos);
+extern template
+std::ostream &operator<<(std::ostream &out, const TokenProcessor<TokenInfo> &tokenInfos);
+extern template
+std::ostream &operator<<(std::ostream &out, const TokenProcessor<FullTokenInfo> &tokenInfos);
 std::ostream &operator<<(std::ostream &out, const FilePathView &filePathView);
 std::ostream &operator<<(std::ostream &out, const NativeFilePathView &nativeFilePathView);
 std::ostream &operator<<(std::ostream &out, const ProjectPartEntry &projectPartEntry);

tests/unit/unittest/highlightingresultreporter-test.cpp

@@ -31,14 +31,14 @@
 #include <clangdocuments.h>
 #include <cursor.h>
 #include <tokeninfocontainer.h>
-#include <tokeninfos.h>
-#include <clangtokeninfosreporter.h>
+#include <tokenprocessor.h>
+#include <clanghighlightingresultreporter.h>
 #include <projectpart.h>
 #include <projects.h>
 #include <unsavedfiles.h>
 
 using ClangBackEnd::Cursor;
-using ClangBackEnd::TokenInfos;
+using ClangBackEnd::TokenProcessor;
 using ClangBackEnd::TokenInfoContainer;
 using ClangBackEnd::HighlightingType;
 using ClangBackEnd::Document;
@@ -61,7 +61,7 @@ struct Data {
                       documents};
 };
 
-class TokenInfosReporter : public ::testing::Test
+class HighlightingResultReporter : public ::testing::Test
 {
 public:
     static void SetUpTestCase();
@@ -88,9 +88,9 @@ QVector<TokenInfoContainer> generateTokenInfos(uint count)
     return container;
 }
 
-TEST_F(TokenInfosReporter, StartAndFinish)
+TEST_F(HighlightingResultReporter, StartAndFinish)
 {
-    auto reporter = new ClangCodeModel::TokenInfosReporter(noTokenInfos());
+    auto reporter = new ClangCodeModel::HighlightingResultReporter(noTokenInfos());
 
     auto future = reporter->start();
 
@@ -98,9 +98,9 @@ TEST_F(TokenInfosReporter, StartAndFinish)
     ASSERT_THAT(future.isFinished(), true);
 }
 
-TEST_F(TokenInfosReporter, ReportNothingIfNothingToReport)
+TEST_F(HighlightingResultReporter, ReportNothingIfNothingToReport)
 {
-    auto reporter = new ClangCodeModel::TokenInfosReporter(generateTokenInfos(0));
+    auto reporter = new ClangCodeModel::HighlightingResultReporter(generateTokenInfos(0));
 
     auto future = reporter->start();
 
@@ -108,9 +108,9 @@ TEST_F(TokenInfosReporter, ReportNothingIfNothingToReport)
     ASSERT_THAT(monitor.resultsReadyCounter(), 0L);
 }
 
-TEST_F(TokenInfosReporter, ReportSingleResultAsOneChunk)
+TEST_F(HighlightingResultReporter, ReportSingleResultAsOneChunk)
 {
-    auto reporter = new ClangCodeModel::TokenInfosReporter(generateTokenInfos(1));
+    auto reporter = new ClangCodeModel::HighlightingResultReporter(generateTokenInfos(1));
     reporter->setChunkSize(1);
 
     auto future = reporter->start();
@@ -119,9 +119,9 @@ TEST_F(TokenInfosReporter, ReportSingleResultAsOneChunk)
     ASSERT_THAT(monitor.resultsReadyCounter(), 1L);
 }
 
-TEST_F(TokenInfosReporter, ReportRestIfChunkSizeNotReached)
+TEST_F(HighlightingResultReporter, ReportRestIfChunkSizeNotReached)
 {
-    auto reporter = new ClangCodeModel::TokenInfosReporter(generateTokenInfos(1));
+    auto reporter = new ClangCodeModel::HighlightingResultReporter(generateTokenInfos(1));
     const int notReachedChunkSize = 100;
     reporter->setChunkSize(notReachedChunkSize);
 
@@ -131,9 +131,9 @@ TEST_F(TokenInfosReporter, ReportRestIfChunkSizeNotReached)
     ASSERT_THAT(monitor.resultsReadyCounter(), 1L);
 }
 
-TEST_F(TokenInfosReporter, ReportChunksWithoutRest)
+TEST_F(HighlightingResultReporter, ReportChunksWithoutRest)
 {
-    auto reporter = new ClangCodeModel::TokenInfosReporter(generateTokenInfos(4));
+    auto reporter = new ClangCodeModel::HighlightingResultReporter(generateTokenInfos(4));
     reporter->setChunkSize(1);
 
     auto future = reporter->start();
@@ -142,9 +142,9 @@ TEST_F(TokenInfosReporter, ReportChunksWithoutRest)
     ASSERT_THAT(monitor.resultsReadyCounter(), 2L);
 }
 
-TEST_F(TokenInfosReporter, ReportSingleChunkAndRest)
+TEST_F(HighlightingResultReporter, ReportSingleChunkAndRest)
 {
-    auto reporter = new ClangCodeModel::TokenInfosReporter(generateTokenInfos(5));
+    auto reporter = new ClangCodeModel::HighlightingResultReporter(generateTokenInfos(5));
     reporter->setChunkSize(2);
 
     auto future = reporter->start();
@@ -153,14 +153,14 @@ TEST_F(TokenInfosReporter, ReportSingleChunkAndRest)
     ASSERT_THAT(monitor.resultsReadyCounter(), 2L);
 }
 
-TEST_F(TokenInfosReporter, ReportCompleteLines)
+TEST_F(HighlightingResultReporter, ReportCompleteLines)
 {
     QVector<TokenInfoContainer> tokenInfos {
         TokenInfoContainer(1, 1, 1, {HighlightingType::Type, {}}),
         TokenInfoContainer(1, 2, 1, {HighlightingType::Type, {}}),
         TokenInfoContainer(2, 1, 1, {HighlightingType::Type, {}}),
     };
-    auto reporter = new ClangCodeModel::TokenInfosReporter(tokenInfos);
+    auto reporter = new ClangCodeModel::HighlightingResultReporter(tokenInfos);
     reporter->setChunkSize(1);
 
     auto future = reporter->start();
@@ -169,14 +169,14 @@ TEST_F(TokenInfosReporter, ReportCompleteLines)
     ASSERT_THAT(monitor.resultsReadyCounter(), 2L);
 }
 
-Data *TokenInfosReporter::d;
+Data *HighlightingResultReporter::d;
 
-void TokenInfosReporter::SetUpTestCase()
+void HighlightingResultReporter::SetUpTestCase()
 {
     d = new Data;
 }
 
-void TokenInfosReporter::TearDownTestCase()
+void HighlightingResultReporter::TearDownTestCase()
 {
     delete d;
     d = nullptr;

tests/unit/unittest/unittest.pro

@@ -90,7 +90,9 @@ SOURCES += \
     processcreator-test.cpp \
     nativefilepath-test.cpp \
     nativefilepathview-test.cpp \
-    mocktimer.cpp
+    mocktimer.cpp \
+    tokenprocessor-test.cpp \
+    highlightingresultreporter-test.cpp
 
 !isEmpty(LIBCLANG_LIBS) {
 SOURCES += \
@@ -146,8 +148,6 @@ SOURCES += \
     sqlitestatement-test.cpp \
     sqlitetable-test.cpp \
     sqlstatementbuilder-test.cpp \
-    tokeninfos-test.cpp \
-    tokeninfosreporter-test.cpp \
     translationunitupdater-test.cpp \
     unsavedfiles-test.cpp \
     unsavedfile-test.cpp \