Clang: add more data to TokenInfo class

Add token name, usr, isDefinition and isDeclaration.

Change-Id: If67bf78c999cb9edd397d0b553b33e5f5f378f8a
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@qt.io>

src/libs/clangsupport/tokeninfocontainer.cpp

@@ -66,8 +66,12 @@ QDebug operator<<(QDebug debug, const TokenInfoContainer &container)
                     << container.column() << ", "
                     << container.length() << ", "
                     << highlightingTypeToCStringLiteral(container.types().mainHighlightingType) << ", "
+                    << container.token() << ", "
+                    << container.typeSpelling() << ", "
                     << container.isIdentifier() << ", "
-                    << container.isIncludeDirectivePath()
+                    << container.isIncludeDirectivePath() << ", "
+                    << container.isDeclaration() << ", "
+                    << container.isDefinition()
                     << ")";
 
     return debug;
@@ -98,8 +102,12 @@ std::ostream &operator<<(std::ostream &os, const TokenInfoContainer &container)
        << container.column() << ", "
        << container.length() << ", "
        << container.types() << ", "
+       << container.token() << ", "
+       << container.typeSpelling() << ", "
        << container.isIdentifier() << ", "
-       << container.isIncludeDirectivePath()
+       << container.isIncludeDirectivePath() << ", "
+       << container.isDeclaration() << ", "
+       << container.isDefinition()
        << ")";
 
     return os;

src/libs/clangsupport/tokeninfocontainer.h

@@ -27,9 +27,12 @@
 
 #include "clangsupport_global.h"
 
+#include <sqlite/utf8string.h>
+
 #include <QDataStream>
 
 #include <iosfwd>
+#include <bitset>
 
 namespace ClangBackEnd {
 
@@ -40,19 +43,41 @@ inline QDataStream &operator>>(QDataStream &in, HighlightingTypes &highlightingT
 inline bool operator==(const MixinHighlightingTypes &first, const MixinHighlightingTypes &second);
 inline bool operator==(const HighlightingTypes &first, const HighlightingTypes &second);
 
+using ByteSizeBitset = std::bitset<8>;
+
+inline QDataStream &operator<<(QDataStream &out, ByteSizeBitset bits);
+inline QDataStream &operator>>(QDataStream &in, ByteSizeBitset &bits);
+
 class TokenInfoContainer
 {
+    enum BitField
+    {
+        Identifier = 0,
+        IncludeDirectivePath = 1,
+        Declaration = 2,
+        Definition = 3,
+        Unused1 = 4,
+        Unused2 = 5,
+        Unused3 = 6,
+        Unused4 = 7,
+    };
 public:
     TokenInfoContainer() = default;
     TokenInfoContainer(uint line, uint column, uint length, HighlightingTypes types,
-                              bool isIdentifier = false, bool isIncludeDirectivePath = false)
+                       const Utf8String &token, const Utf8String &typeSpelling,
+                       bool isIdentifier = false, bool isIncludeDirectivePath = false,
+                       bool isDeclaration = false, bool isDefinition = false)
         : line_(line),
           column_(column),
           length_(length),
           types_(types),
-          isIdentifier_(isIdentifier),
+          token_(token),
-          isIncludeDirectivePath_(isIncludeDirectivePath)
+          typeSpelling_(typeSpelling)
     {
+        bitFields_.set(BitField::Identifier, isIdentifier);
+        bitFields_.set(BitField::IncludeDirectivePath, isIncludeDirectivePath);
+        bitFields_.set(BitField::Declaration, isDeclaration);
+        bitFields_.set(BitField::Definition, isDefinition);
     }
 
     TokenInfoContainer(uint line, uint column, uint length, HighlightingType type)
@@ -90,22 +115,44 @@ public:
 
     bool isIdentifier() const
     {
-        return isIdentifier_;
+        return bitFields_[BitField::Identifier];
     }
 
     bool isIncludeDirectivePath() const
     {
-        return isIncludeDirectivePath_;
+        return bitFields_[BitField::IncludeDirectivePath];
     }
 
+    bool isDeclaration() const
+    {
+        return bitFields_[BitField::Declaration];
+    }
+
+    bool isDefinition() const
+    {
+        return bitFields_[BitField::Definition];
+    }
+
+    const Utf8String &token() const
+    {
+        return token_;
+    }
+
+    const Utf8String &typeSpelling() const
+    {
+        return typeSpelling_;
+    }
+
+
     friend QDataStream &operator<<(QDataStream &out, const TokenInfoContainer &container)
     {
         out << container.line_;
         out << container.column_;
         out << container.length_;
         out << container.types_;
-        out << container.isIdentifier_;
+        out << container.token_;
-        out << container.isIncludeDirectivePath_;
+        out << container.typeSpelling_;
+        out << container.bitFields_;
 
         return out;
     }
@@ -116,8 +163,9 @@ public:
         in >> container.column_;
         in >> container.length_;
         in >> container.types_;
-        in >> container.isIdentifier_;
+        in >> container.token_ ;
-        in >> container.isIncludeDirectivePath_;
+        in >> container.typeSpelling_;
+        in >> container.bitFields_;
 
         return in;
     }
@@ -128,8 +176,7 @@ public:
             && first.column_ == second.column_
             && first.length_ == second.length_
             && first.types_ == second.types_
-            && first.isIdentifier_ == second.isIdentifier_
+            && first.bitFields_ == second.bitFields_;
-            && first.isIncludeDirectivePath_ == second.isIncludeDirectivePath_;
     }
 
 private:
@@ -137,8 +184,9 @@ private:
     uint column_ = 0;
     uint length_ = 0;
     HighlightingTypes types_;
-    bool isIdentifier_ = false;
+    Utf8String token_;
-    bool isIncludeDirectivePath_ = false;
+    Utf8String typeSpelling_;
+    ByteSizeBitset bitFields_;
 };
 
 inline QDataStream &operator<<(QDataStream &out, HighlightingType highlightingType)
@@ -200,6 +248,21 @@ inline bool operator==(const HighlightingTypes &first, const HighlightingTypes &
         && first.mixinHighlightingTypes == second.mixinHighlightingTypes;
 }
 
+inline QDataStream &operator<<(QDataStream &out, ByteSizeBitset bits)
+{
+    // Narrow unsigned long to uint8_t
+    out << static_cast<uint8_t>(bits.to_ulong());
+    return out;
+}
+
+inline QDataStream &operator>>(QDataStream &in, ByteSizeBitset &bits)
+{
+    uint8_t byteValue;
+    in >> byteValue;
+    bits = ByteSizeBitset(byteValue);
+    return in;
+}
+
 CLANGSUPPORT_EXPORT QDebug operator<<(QDebug debug, const TokenInfoContainer &container);
 CLANGSUPPORT_EXPORT std::ostream &operator<<(std::ostream &os, HighlightingType highlightingType);
 CLANGSUPPORT_EXPORT std::ostream &operator<<(std::ostream &os, HighlightingTypes types);

src/tools/clangbackend/source/tokeninfo.cpp

@@ -40,9 +40,9 @@
 namespace ClangBackEnd {
 
 TokenInfo::TokenInfo(const CXCursor &cxCursor,
-                                   CXToken *cxToken,
+                     CXToken *cxToken,
-                                   CXTranslationUnit cxTranslationUnit,
+                     CXTranslationUnit cxTranslationUnit,
-                                   std::vector<CXSourceRange> &currentOutputArgumentRanges)
+                     std::vector<CXSourceRange> &currentOutputArgumentRanges)
     : m_currentOutputArgumentRanges(&currentOutputArgumentRanges),
       m_originalCursor(cxCursor)
 {
@@ -115,8 +115,9 @@ bool TokenInfo::hasFunctionArguments() const
 
 TokenInfo::operator TokenInfoContainer() const
 {
-    return TokenInfoContainer(m_line, m_column, m_length, m_types, m_isIdentifier,
+    return TokenInfoContainer(m_line, m_column, m_length, m_types, m_token, m_typeSpelling,
-                                     m_isInclusion);
+                              m_isIdentifier, m_isInclusion,
+                              m_isDeclaration, m_isDefinition);
 }
 
 namespace {
@@ -451,6 +452,13 @@ void TokenInfo::collectKinds(CXTranslationUnit cxTranslationUnit,
     auto cxTokenKind = clang_getTokenKind(*cxToken);
 
     m_types = HighlightingTypes();
+    m_token = ClangString(clang_getTokenSpelling(cxTranslationUnit, *cxToken));
+    m_typeSpelling = cursor.type().utf8Spelling();
+
+    if (cxTokenKind == CXToken_Identifier) {
+        m_isDeclaration = cursor.isDeclaration();
+        m_isDefinition = cursor.isDefinition();
+    }
 
     switch (cxTokenKind) {
         case CXToken_Keyword:     m_types.mainHighlightingType = highlightingTypeForKeyword(cxTranslationUnit, cxToken, m_originalCursor); break;

src/tools/clangbackend/source/tokeninfo.h

@@ -25,10 +25,11 @@
 
 #pragma once
 
-#include <clangsupport_global.h>
+#include "clangsupport_global.h"
-#include <tokeninfocontainer.h>
-
 #include "cursor.h"
+#include "tokeninfocontainer.h"
+
+#include <sqlite/utf8string.h>
 
 #include <clang-c/Index.h>
 
@@ -45,9 +46,9 @@ class TokenInfo
 
 public:
     TokenInfo(const CXCursor &cxCursor,
-                     CXToken *cxToken,
+              CXToken *cxToken,
-                     CXTranslationUnit cxTranslationUnit,
+              CXTranslationUnit cxTranslationUnit,
-                     std::vector<CXSourceRange> &m_currentOutputArgumentRanges);
+              std::vector<CXSourceRange> &m_currentOutputArgumentRanges);
     TokenInfo(uint m_line, uint m_column, uint m_length, HighlightingTypes m_types);
     TokenInfo(uint m_line, uint m_column, uint m_length, HighlightingType type);
 
@@ -90,8 +91,12 @@ private:
     uint m_length;
     uint m_offset = 0;
     HighlightingTypes m_types;
+    Utf8String m_token;
+    Utf8String m_typeSpelling;
     bool m_isIdentifier = false;
     bool m_isInclusion = false;
+    bool m_isDeclaration = false;
+    bool m_isDefinition = false;
 };
 
 

tests/unit/unittest/clangcodemodelserver-test.cpp

@@ -630,7 +630,8 @@ void ClangCodeModelServer::expectDocumentAnnotationsChangedForFileBWithSpecificH
     types.mainHighlightingType = ClangBackEnd::HighlightingType::Function;
     types.mixinHighlightingTypes.push_back(ClangBackEnd::HighlightingType::Declaration);
     types.mixinHighlightingTypes.push_back(ClangBackEnd::HighlightingType::FunctionDefinition);
-    const TokenInfoContainer tokenInfo(1, 6, 8, types, true);
+    const TokenInfoContainer tokenInfo(1, 6, 8, types, Utf8String("function", 8),
+                                       Utf8String("void (int)", 10), true, false, true, true);
 
     EXPECT_CALL(mockClangCodeModelClient,
                 documentAnnotationsChanged(