From 11d002968ea22c6df041a564a38e929c5ad1d1d2 Mon Sep 17 00:00:00 2001 From: Ivan Donchevskii Date: Tue, 28 Nov 2017 14:12:56 +0100 Subject: [PATCH] Clang: add more data to TokenInfo class Add token name, usr, isDefinition and isDeclaration. Change-Id: If67bf78c999cb9edd397d0b553b33e5f5f378f8a Reviewed-by: Nikolai Kosjar --- src/libs/clangsupport/tokeninfocontainer.cpp | 12 ++- src/libs/clangsupport/tokeninfocontainer.h | 89 ++++++++++++++++--- src/tools/clangbackend/source/tokeninfo.cpp | 18 ++-- src/tools/clangbackend/source/tokeninfo.h | 17 ++-- .../unittest/clangcodemodelserver-test.cpp | 3 +- 5 files changed, 112 insertions(+), 27 deletions(-) diff --git a/src/libs/clangsupport/tokeninfocontainer.cpp b/src/libs/clangsupport/tokeninfocontainer.cpp index 5ac4dcf6aeb..beca0c5f9f1 100644 --- a/src/libs/clangsupport/tokeninfocontainer.cpp +++ b/src/libs/clangsupport/tokeninfocontainer.cpp @@ -66,8 +66,12 @@ QDebug operator<<(QDebug debug, const TokenInfoContainer &container) << container.column() << ", " << container.length() << ", " << highlightingTypeToCStringLiteral(container.types().mainHighlightingType) << ", " + << container.token() << ", " + << container.typeSpelling() << ", " << container.isIdentifier() << ", " - << container.isIncludeDirectivePath() + << container.isIncludeDirectivePath() << ", " + << container.isDeclaration() << ", " + << container.isDefinition() << ")"; return debug; @@ -98,8 +102,12 @@ std::ostream &operator<<(std::ostream &os, const TokenInfoContainer &container) << container.column() << ", " << container.length() << ", " << container.types() << ", " + << container.token() << ", " + << container.typeSpelling() << ", " << container.isIdentifier() << ", " - << container.isIncludeDirectivePath() + << container.isIncludeDirectivePath() << ", " + << container.isDeclaration() << ", " + << container.isDefinition() << ")"; return os; diff --git a/src/libs/clangsupport/tokeninfocontainer.h b/src/libs/clangsupport/tokeninfocontainer.h index c3041cb4379..b5007b0b1d3 100644 --- a/src/libs/clangsupport/tokeninfocontainer.h +++ b/src/libs/clangsupport/tokeninfocontainer.h @@ -27,9 +27,12 @@ #include "clangsupport_global.h" +#include + #include #include +#include namespace ClangBackEnd { @@ -40,19 +43,41 @@ inline QDataStream &operator>>(QDataStream &in, HighlightingTypes &highlightingT inline bool operator==(const MixinHighlightingTypes &first, const MixinHighlightingTypes &second); inline bool operator==(const HighlightingTypes &first, const HighlightingTypes &second); +using ByteSizeBitset = std::bitset<8>; + +inline QDataStream &operator<<(QDataStream &out, ByteSizeBitset bits); +inline QDataStream &operator>>(QDataStream &in, ByteSizeBitset &bits); + class TokenInfoContainer { + enum BitField + { + Identifier = 0, + IncludeDirectivePath = 1, + Declaration = 2, + Definition = 3, + Unused1 = 4, + Unused2 = 5, + Unused3 = 6, + Unused4 = 7, + }; public: TokenInfoContainer() = default; TokenInfoContainer(uint line, uint column, uint length, HighlightingTypes types, - bool isIdentifier = false, bool isIncludeDirectivePath = false) + const Utf8String &token, const Utf8String &typeSpelling, + bool isIdentifier = false, bool isIncludeDirectivePath = false, + bool isDeclaration = false, bool isDefinition = false) : line_(line), column_(column), length_(length), types_(types), - isIdentifier_(isIdentifier), - isIncludeDirectivePath_(isIncludeDirectivePath) + token_(token), + typeSpelling_(typeSpelling) { + bitFields_.set(BitField::Identifier, isIdentifier); + bitFields_.set(BitField::IncludeDirectivePath, isIncludeDirectivePath); + bitFields_.set(BitField::Declaration, isDeclaration); + bitFields_.set(BitField::Definition, isDefinition); } TokenInfoContainer(uint line, uint column, uint length, HighlightingType type) @@ -90,22 +115,44 @@ public: bool isIdentifier() const { - return isIdentifier_; + return bitFields_[BitField::Identifier]; } bool isIncludeDirectivePath() const { - return isIncludeDirectivePath_; + return bitFields_[BitField::IncludeDirectivePath]; } + bool isDeclaration() const + { + return bitFields_[BitField::Declaration]; + } + + bool isDefinition() const + { + return bitFields_[BitField::Definition]; + } + + const Utf8String &token() const + { + return token_; + } + + const Utf8String &typeSpelling() const + { + return typeSpelling_; + } + + friend QDataStream &operator<<(QDataStream &out, const TokenInfoContainer &container) { out << container.line_; out << container.column_; out << container.length_; out << container.types_; - out << container.isIdentifier_; - out << container.isIncludeDirectivePath_; + out << container.token_; + out << container.typeSpelling_; + out << container.bitFields_; return out; } @@ -116,8 +163,9 @@ public: in >> container.column_; in >> container.length_; in >> container.types_; - in >> container.isIdentifier_; - in >> container.isIncludeDirectivePath_; + in >> container.token_ ; + in >> container.typeSpelling_; + in >> container.bitFields_; return in; } @@ -128,8 +176,7 @@ public: && first.column_ == second.column_ && first.length_ == second.length_ && first.types_ == second.types_ - && first.isIdentifier_ == second.isIdentifier_ - && first.isIncludeDirectivePath_ == second.isIncludeDirectivePath_; + && first.bitFields_ == second.bitFields_; } private: @@ -137,8 +184,9 @@ private: uint column_ = 0; uint length_ = 0; HighlightingTypes types_; - bool isIdentifier_ = false; - bool isIncludeDirectivePath_ = false; + Utf8String token_; + Utf8String typeSpelling_; + ByteSizeBitset bitFields_; }; inline QDataStream &operator<<(QDataStream &out, HighlightingType highlightingType) @@ -200,6 +248,21 @@ inline bool operator==(const HighlightingTypes &first, const HighlightingTypes & && first.mixinHighlightingTypes == second.mixinHighlightingTypes; } +inline QDataStream &operator<<(QDataStream &out, ByteSizeBitset bits) +{ + // Narrow unsigned long to uint8_t + out << static_cast(bits.to_ulong()); + return out; +} + +inline QDataStream &operator>>(QDataStream &in, ByteSizeBitset &bits) +{ + uint8_t byteValue; + in >> byteValue; + bits = ByteSizeBitset(byteValue); + return in; +} + CLANGSUPPORT_EXPORT QDebug operator<<(QDebug debug, const TokenInfoContainer &container); CLANGSUPPORT_EXPORT std::ostream &operator<<(std::ostream &os, HighlightingType highlightingType); CLANGSUPPORT_EXPORT std::ostream &operator<<(std::ostream &os, HighlightingTypes types); diff --git a/src/tools/clangbackend/source/tokeninfo.cpp b/src/tools/clangbackend/source/tokeninfo.cpp index 63cfb9c679d..51fde96e66a 100644 --- a/src/tools/clangbackend/source/tokeninfo.cpp +++ b/src/tools/clangbackend/source/tokeninfo.cpp @@ -40,9 +40,9 @@ namespace ClangBackEnd { TokenInfo::TokenInfo(const CXCursor &cxCursor, - CXToken *cxToken, - CXTranslationUnit cxTranslationUnit, - std::vector ¤tOutputArgumentRanges) + CXToken *cxToken, + CXTranslationUnit cxTranslationUnit, + std::vector ¤tOutputArgumentRanges) : m_currentOutputArgumentRanges(¤tOutputArgumentRanges), m_originalCursor(cxCursor) { @@ -115,8 +115,9 @@ bool TokenInfo::hasFunctionArguments() const TokenInfo::operator TokenInfoContainer() const { - return TokenInfoContainer(m_line, m_column, m_length, m_types, m_isIdentifier, - m_isInclusion); + return TokenInfoContainer(m_line, m_column, m_length, m_types, m_token, m_typeSpelling, + m_isIdentifier, m_isInclusion, + m_isDeclaration, m_isDefinition); } namespace { @@ -451,6 +452,13 @@ void TokenInfo::collectKinds(CXTranslationUnit cxTranslationUnit, auto cxTokenKind = clang_getTokenKind(*cxToken); m_types = HighlightingTypes(); + m_token = ClangString(clang_getTokenSpelling(cxTranslationUnit, *cxToken)); + m_typeSpelling = cursor.type().utf8Spelling(); + + if (cxTokenKind == CXToken_Identifier) { + m_isDeclaration = cursor.isDeclaration(); + m_isDefinition = cursor.isDefinition(); + } switch (cxTokenKind) { case CXToken_Keyword: m_types.mainHighlightingType = highlightingTypeForKeyword(cxTranslationUnit, cxToken, m_originalCursor); break; diff --git a/src/tools/clangbackend/source/tokeninfo.h b/src/tools/clangbackend/source/tokeninfo.h index 951a6f99a6f..a980e2e3e85 100644 --- a/src/tools/clangbackend/source/tokeninfo.h +++ b/src/tools/clangbackend/source/tokeninfo.h @@ -25,10 +25,11 @@ #pragma once -#include -#include - +#include "clangsupport_global.h" #include "cursor.h" +#include "tokeninfocontainer.h" + +#include #include @@ -45,9 +46,9 @@ class TokenInfo public: TokenInfo(const CXCursor &cxCursor, - CXToken *cxToken, - CXTranslationUnit cxTranslationUnit, - std::vector &m_currentOutputArgumentRanges); + CXToken *cxToken, + CXTranslationUnit cxTranslationUnit, + std::vector &m_currentOutputArgumentRanges); TokenInfo(uint m_line, uint m_column, uint m_length, HighlightingTypes m_types); TokenInfo(uint m_line, uint m_column, uint m_length, HighlightingType type); @@ -90,8 +91,12 @@ private: uint m_length; uint m_offset = 0; HighlightingTypes m_types; + Utf8String m_token; + Utf8String m_typeSpelling; bool m_isIdentifier = false; bool m_isInclusion = false; + bool m_isDeclaration = false; + bool m_isDefinition = false; }; diff --git a/tests/unit/unittest/clangcodemodelserver-test.cpp b/tests/unit/unittest/clangcodemodelserver-test.cpp index 9a3b36d8550..bb20a5fa353 100644 --- a/tests/unit/unittest/clangcodemodelserver-test.cpp +++ b/tests/unit/unittest/clangcodemodelserver-test.cpp @@ -630,7 +630,8 @@ void ClangCodeModelServer::expectDocumentAnnotationsChangedForFileBWithSpecificH types.mainHighlightingType = ClangBackEnd::HighlightingType::Function; types.mixinHighlightingTypes.push_back(ClangBackEnd::HighlightingType::Declaration); types.mixinHighlightingTypes.push_back(ClangBackEnd::HighlightingType::FunctionDefinition); - const TokenInfoContainer tokenInfo(1, 6, 8, types, true); + const TokenInfoContainer tokenInfo(1, 6, 8, types, Utf8String("function", 8), + Utf8String("void (int)", 10), true, false, true, true); EXPECT_CALL(mockClangCodeModelClient, documentAnnotationsChanged(