C++: Store comment tokens in the translation unit.

Change-Id: I904123bdbbd675e9f018236c1fed13528d83f87f
Reviewed-on: http://codereview.qt.nokia.com/3481
Reviewed-by: Roberto Raggi <roberto.raggi@nokia.com>

src/libs/3rdparty/cplusplus/TranslationUnit.cpp

@@ -48,6 +48,7 @@ TranslationUnit::TranslationUnit(Control *control, const StringLiteral *fileId)
       _flags(0)
 {
     _tokens = new std::vector<Token>();
+    _comments = new std::vector<Token>();
     _previousTranslationUnit = control->switchTranslationUnit(this);
     _pool = new MemoryPool();
 }
@@ -56,6 +57,7 @@ TranslationUnit::~TranslationUnit()
 {
     (void) _control->switchTranslationUnit(_previousTranslationUnit);
     delete _tokens;
+    delete _comments;
     delete _pool;
 }
 
@@ -121,6 +123,12 @@ const char *TranslationUnit::spell(unsigned index) const
     return _tokens->at(index).spell();
 }
 
+unsigned TranslationUnit::commentCount() const
+{ return _comments->size(); }
+
+const Token &TranslationUnit::commentAt(unsigned index) const
+{ return _comments->at(index); }
+
 const Identifier *TranslationUnit::identifier(unsigned index) const
 { return _tokens->at(index).identifier; }
 
@@ -159,6 +167,7 @@ void TranslationUnit::tokenize()
     lex.setQtMocRunEnabled(f._qtMocRunEnabled);
     lex.setCxxOxEnabled(f._cxx0xEnabled);
     lex.setObjCEnabled(f._objCEnabled);
+    lex.setScanCommentTokens(true);
 
     std::stack<unsigned> braces;
     _tokens->push_back(Token()); // the first token needs to be invalid!
@@ -212,6 +221,9 @@ void TranslationUnit::tokenize()
             const unsigned open_brace_index = braces.top();
             braces.pop();
             (*_tokens)[open_brace_index].close_brace = _tokens->size();
+        } else if (tk.isComment()) {
+            _comments->push_back(tk);
+            continue; // comments are not in the regular token stream
         }
         tk.f.generated = generated;
         _tokens->push_back(tk);
@@ -475,6 +487,8 @@ void TranslationUnit::release()
     resetAST();
     delete _tokens;
     _tokens = 0;
+    delete _comments;
+    _comments = 0;
 }
 
 

src/libs/3rdparty/cplusplus/TranslationUnit.h

@@ -57,6 +57,9 @@ public:
     int tokenKind(unsigned index) const;
     const char *spell(unsigned index) const;
 
+    unsigned commentCount() const;
+    const Token &commentAt(unsigned index) const;
+
     unsigned matchingBrace(unsigned index) const;
     const Identifier *identifier(unsigned index) const;
     const Literal *literal(unsigned index) const;
@@ -164,6 +167,7 @@ private:
     const char *_firstSourceChar;
     const char *_lastSourceChar;
     std::vector<Token> *_tokens;
+    std::vector<Token> *_comments;
     std::vector<unsigned> _lineOffsets;
     std::vector<PPLine> _ppLines;
     MemoryPool *_pool;