C++: Don't strip comments from preprocessed source code.

Change-Id: I3a4817d36b9c724abca504c42914a73f97ab34c7
Reviewed-on: http://codereview.qt.nokia.com/3480
Reviewed-by: Roberto Raggi <roberto.raggi@nokia.com>

src/libs/cplusplus/FastPreprocessor.cpp

@@ -45,6 +45,7 @@ FastPreprocessor::FastPreprocessor(const Snapshot &snapshot)
 QByteArray FastPreprocessor::run(QString fileName, const QString &source)
 {
     _preproc.setExpandMacros(false);
+    _preproc.setKeepComments(true);
 
     if (Document::Ptr doc = _snapshot.document(fileName)) {
         _merged.insert(fileName);

src/libs/cplusplus/pp-engine.cpp

@@ -461,7 +461,8 @@ Preprocessor::Preprocessor(Client *client, Environment *env)
       _dot(_tokens.end()),
       _result(0),
       _markGeneratedTokens(false),
-      _expandMacros(true)
+      _expandMacros(true),
+      _keepComments(false)
 {
     resetIfLevel ();
 }
@@ -558,12 +559,24 @@ void Preprocessor::setExpandMacros(bool expandMacros)
     _expandMacros = expandMacros;
 }
 
+bool Preprocessor::keepComments() const
+{
+    return _keepComments;
+}
+
+void Preprocessor::setKeepComments(bool keepComments)
+{
+    _keepComments = keepComments;
+}
+
 Preprocessor::State Preprocessor::createStateFromSource(const QByteArray &source) const
 {
     State state;
     state.source = source;
     Lexer lex(state.source.constBegin(), state.source.constEnd());
     lex.setScanKeywords(false);
+    if (_keepComments)
+        lex.setScanCommentTokens(true);
     Token tok;
     do {
         lex(&tok);
@@ -578,7 +591,11 @@ void Preprocessor::processNewline(bool force)
     if (_dot != _tokens.constBegin()) {
         TokenIterator prevTok = _dot - 1;
 
-        if (prevTok->isLiteral()) {
+        // line changes due to multi-line tokens
+        if (prevTok->isLiteral()
+                || (_keepComments
+                    && (prevTok->kind() == T_COMMENT
+                        || prevTok->kind() == T_DOXY_COMMENT))) {
             const char *ptr = _source.constBegin() + prevTok->begin();
             const char *end = ptr + prevTok->length();
 

src/libs/cplusplus/pp-engine.h

@@ -79,6 +79,9 @@ public:
     bool expandMacros() const;
     void setExpandMacros(bool expandMacros);
 
+    bool keepComments() const;
+    void setKeepComments(bool keepComments);
+
 private:
     enum { MAX_LEVEL = 512 };
 
@@ -197,6 +200,7 @@ private:
 
     QString _originalSource;
     bool _expandMacros;
+    bool _keepComments;
 };
 
 } // namespace CPlusPlus

src/plugins/cpptools/cppmodelmanager.cpp

@@ -173,7 +173,9 @@ CppPreprocessor::CppPreprocessor(QPointer<CppModelManager> modelManager)
       m_modelManager(modelManager),
       preprocess(this, &env),
       m_revision(0)
-{ }
+{
+    preprocess.setKeepComments(true);
+}
 
 #else