diff --git a/src/libs/3rdparty/cplusplus/Token.h b/src/libs/3rdparty/cplusplus/Token.h index 37a7a725c1c..a30474adc67 100644 --- a/src/libs/3rdparty/cplusplus/Token.h +++ b/src/libs/3rdparty/cplusplus/Token.h @@ -321,13 +321,29 @@ public: public: struct Flags { + // The token kind. unsigned kind : 8; + // The token starts a new line. unsigned newline : 1; + // The token is preceeded by whitespace(s). unsigned whitespace : 1; + // The token is joined with the previous one. unsigned joined : 1; + // The token originates from a macro expansion. unsigned expanded : 1; + // The token originates from a macro expansion and does not correspond to an + // argument that went through substitution. Notice the example: + // + // #define FOO(a, b) a + b; + // FOO(1, 2) + // + // After preprocessing we would expect the following tokens: 1 + 2; + // Tokens '1', '+', '2', and ';' are all expanded. However only tokens '+' and ';' + // are generated. unsigned generated : 1; + // Unused... unsigned pad : 3; + // The token lenght. unsigned length : 16; }; union { diff --git a/src/libs/3rdparty/cplusplus/TranslationUnit.cpp b/src/libs/3rdparty/cplusplus/TranslationUnit.cpp index 3a083d6e3f2..a26a23eabce 100644 --- a/src/libs/3rdparty/cplusplus/TranslationUnit.cpp +++ b/src/libs/3rdparty/cplusplus/TranslationUnit.cpp @@ -27,8 +27,10 @@ #include "Literals.h" #include "DiagnosticClient.h" #include +#include #include #include +#include #ifdef _MSC_VER # define va_copy(dst, src) ((dst) = (src)) @@ -176,27 +178,84 @@ void TranslationUnit::tokenize() pushPreprocessorLine(0, 1, fileId()); const Identifier *lineId = control()->identifier("line"); - const Identifier *genId = control()->identifier("gen"); + const Identifier *expansionId = control()->identifier("expansion"); + const Identifier *beginId = control()->identifier("begin"); + const Identifier *endId = control()->identifier("end"); + + // We need to track information about the expanded tokens. A vector with an addition + // explicit index control is used instead of queue mainly for performance reasons. + std::vector > lineColumn; + unsigned lineColumnIdx = 0; - bool generated = false; Token tk; do { lex(&tk); - _Lrecognize: + _Lrecognize: if (tk.is(T_POUND) && tk.newline()) { unsigned offset = tk.offset; lex(&tk); - if (! tk.f.newline && tk.is(T_IDENTIFIER) && tk.identifier == genId) { - // it's a gen directive. + if (! tk.f.newline && tk.is(T_IDENTIFIER) && tk.identifier == expansionId) { + // It's an expansion mark. lex(&tk); - if (! tk.f.newline && tk.is(T_TRUE)) { - lex(&tk); - generated = true; - } else { - generated = false; + if (!tk.f.newline && tk.is(T_IDENTIFIER)) { + if (tk.identifier == beginId) { + // Start of a macro expansion section. + lex(&tk); + + // Gather where the expansion happens and its length. + unsigned macroOffset = static_cast(strtoul(tk.spell(), 0, 0)); + lex(&tk); + lex(&tk); // Skip the separating comma + unsigned macroLength = static_cast(strtoul(tk.spell(), 0, 0)); + lex(&tk); + + // NOTE: We are currently not using the macro offset and length. They + // are kept here for now because of future use. + Q_UNUSED(macroOffset) + Q_UNUSED(macroLength) + + // Now we need to gather the real line and columns from the upcoming + // tokens. But notice this is only relevant for tokens which are expanded + // but not generated. + while (tk.isNot(T_EOF_SYMBOL) && !tk.f.newline) { + // When we get a ~ it means there's a number of generated tokens + // following. Otherwise, we have actual data. + if (tk.is(T_TILDE)) { + lex(&tk); + + // Get the total number of generated tokens and specifiy "null" + // information for them. + unsigned totalGenerated = + static_cast(strtoul(tk.spell(), 0, 0)); + const std::size_t previousSize = lineColumn.size(); + lineColumn.resize(previousSize + totalGenerated); + std::fill(lineColumn.begin() + previousSize, + lineColumn.end(), + std::make_pair(0, 0)); + + lex(&tk); + } else if (tk.is(T_NUMERIC_LITERAL)) { + unsigned line = static_cast(strtoul(tk.spell(), 0, 0)); + lex(&tk); + lex(&tk); // Skip the separating colon + unsigned column = static_cast(strtoul(tk.spell(), 0, 0)); + + // Store line and column for this non-generated token. + lineColumn.push_back(std::make_pair(line, column)); + + lex(&tk); + } + } + } else if (tk.identifier == endId) { + // End of a macro expansion. + lineColumn.clear(); + lineColumnIdx = 0; + + lex(&tk); + } } } else { if (! tk.f.newline && tk.is(T_IDENTIFIER) && tk.identifier == lineId) @@ -211,9 +270,9 @@ void TranslationUnit::tokenize() lex(&tk); } } + while (tk.isNot(T_EOF_SYMBOL) && ! tk.f.newline) + lex(&tk); } - while (tk.isNot(T_EOF_SYMBOL) && ! tk.f.newline) - lex(&tk); goto _Lrecognize; } else if (tk.f.kind == T_LBRACE) { braces.push(_tokens->size()); @@ -225,7 +284,24 @@ void TranslationUnit::tokenize() _comments->push_back(tk); continue; // comments are not in the regular token stream } - tk.f.generated = generated; + + bool currentExpanded = false; + bool currentGenerated = false; + + if (!lineColumn.empty() && lineColumnIdx < lineColumn.size()) { + currentExpanded = true; + const std::pair &p = lineColumn[lineColumnIdx]; + if (p.first) + _expandedLineColumn.insert(std::make_pair(tk.offset, p)); + else + currentGenerated = true; + + ++lineColumnIdx; + } + + tk.f.expanded = currentExpanded; + tk.f.generated = currentGenerated; + _tokens->push_back(tk); } while (tk.f.kind); @@ -355,12 +431,32 @@ void TranslationUnit::getPosition(unsigned tokenOffset, unsigned *column, const StringLiteral **fileName) const { - unsigned lineNumber = findLineNumber(tokenOffset); - unsigned columnNumber = findColumnNumber(tokenOffset, lineNumber); - const PPLine ppLine = findPreprocessorLine(tokenOffset); + unsigned lineNumber = 0; + unsigned columnNumber = 0; + const StringLiteral *file = 0; - lineNumber -= findLineNumber(ppLine.offset) + 1; - lineNumber += ppLine.line; + // If this token is expanded we already have the information directly from the expansion + // section header. Otherwise, we need to calculate it. + std::map >::const_iterator it = + _expandedLineColumn.find(tokenOffset); + if (it != _expandedLineColumn.end()) { + lineNumber = it->second.first; + columnNumber = it->second.second + 1; + file = _fileId; + } else { + // Identify line within the entire translation unit. + lineNumber = findLineNumber(tokenOffset); + + // Identify column. + columnNumber = findColumnNumber(tokenOffset, lineNumber); + + // Adjust the line in regards to the preprocessing markers. + const PPLine ppLine = findPreprocessorLine(tokenOffset); + lineNumber -= findLineNumber(ppLine.offset) + 1; + lineNumber += ppLine.line; + + file = ppLine.fileName; + } if (line) *line = lineNumber; @@ -369,7 +465,7 @@ void TranslationUnit::getPosition(unsigned tokenOffset, *column = columnNumber; if (fileName) - *fileName = ppLine.fileName; + *fileName = file; } bool TranslationUnit::blockErrors(bool block) diff --git a/src/libs/3rdparty/cplusplus/TranslationUnit.h b/src/libs/3rdparty/cplusplus/TranslationUnit.h index a98675b7a84..95832f43281 100644 --- a/src/libs/3rdparty/cplusplus/TranslationUnit.h +++ b/src/libs/3rdparty/cplusplus/TranslationUnit.h @@ -27,7 +27,7 @@ #include "DiagnosticClient.h" #include #include - +#include namespace CPlusPlus { @@ -170,6 +170,7 @@ private: std::vector *_comments; std::vector _lineOffsets; std::vector _ppLines; + std::map > _expandedLineColumn; // TODO: Replace this for a hash MemoryPool *_pool; AST *_ast; TranslationUnit *_previousTranslationUnit; diff --git a/src/libs/cplusplus/FastPreprocessor.cpp b/src/libs/cplusplus/FastPreprocessor.cpp index 5e2b22354d2..801af3e235c 100644 --- a/src/libs/cplusplus/FastPreprocessor.cpp +++ b/src/libs/cplusplus/FastPreprocessor.cpp @@ -60,7 +60,7 @@ QByteArray FastPreprocessor::run(QString fileName, const QString &source) return preprocessed; } -void FastPreprocessor::sourceNeeded(QString &fileName, IncludeType, unsigned) +void FastPreprocessor::sourceNeeded(unsigned, QString &fileName, IncludeType) { mergeEnvironment(fileName); } void FastPreprocessor::mergeEnvironment(const QString &fileName) diff --git a/src/libs/cplusplus/FastPreprocessor.h b/src/libs/cplusplus/FastPreprocessor.h index bcd9075cc22..b2f8cc5611f 100644 --- a/src/libs/cplusplus/FastPreprocessor.h +++ b/src/libs/cplusplus/FastPreprocessor.h @@ -59,18 +59,19 @@ public: QByteArray run(QString fileName, const QString &source); // CPlusPlus::Client - virtual void sourceNeeded(QString &fileName, IncludeType, unsigned); + virtual void sourceNeeded(unsigned, QString &fileName, IncludeType); virtual void macroAdded(const Macro &) {} - virtual void passedMacroDefinitionCheck(unsigned, const Macro &) {} + virtual void passedMacroDefinitionCheck(unsigned, unsigned, const Macro &) {} virtual void failedMacroDefinitionCheck(unsigned, const ByteArrayRef &) {} - virtual void startExpandingMacro(unsigned, - const Macro &, - const ByteArrayRef &, - const QVector &) {} + virtual void notifyMacroReference(unsigned, unsigned, const Macro &) {} + virtual void startExpandingMacro(unsigned, + unsigned, + const Macro &, + const QVector &) {} virtual void stopExpandingMacro(unsigned, const Macro &) {} virtual void startSkippingBlocks(unsigned) {} diff --git a/src/libs/cplusplus/PPToken.cpp b/src/libs/cplusplus/PPToken.cpp index 3867451c261..948049c4dba 100644 --- a/src/libs/cplusplus/PPToken.cpp +++ b/src/libs/cplusplus/PPToken.cpp @@ -23,10 +23,10 @@ int ByteArrayRef::count(char ch) const return num; } -void Internal::PPToken::squeeze() +void Internal::PPToken::squeezeSource() { - if (isValid()) { - m_src = m_src.mid(offset, length()); + if (hasSource()) { + m_src = m_src.mid(offset, f.length); m_src.squeeze(); offset = 0; } diff --git a/src/libs/cplusplus/PPToken.h b/src/libs/cplusplus/PPToken.h index adcdda96377..98600811a63 100644 --- a/src/libs/cplusplus/PPToken.h +++ b/src/libs/cplusplus/PPToken.h @@ -96,6 +96,11 @@ public: const QByteArray &source() const { return m_src; } + bool hasSource() const + { return !m_src.isEmpty(); } + + void squeezeSource(); + const char *bufferStart() const { return m_src.constData(); } @@ -105,11 +110,6 @@ public: ByteArrayRef asByteArrayRef() const { return ByteArrayRef(&m_src, offset, length()); } - bool isValid() const - { return !m_src.isEmpty(); } - - void squeeze(); - private: QByteArray m_src; }; diff --git a/src/libs/cplusplus/PreprocessorClient.h b/src/libs/cplusplus/PreprocessorClient.h index af60a562c72..264ce3295c3 100644 --- a/src/libs/cplusplus/PreprocessorClient.h +++ b/src/libs/cplusplus/PreprocessorClient.h @@ -80,24 +80,23 @@ public: virtual void macroAdded(const Macro ¯o) = 0; - virtual void passedMacroDefinitionCheck(unsigned offset, const Macro ¯o) = 0; + virtual void passedMacroDefinitionCheck(unsigned offset, unsigned line, const Macro ¯o) = 0; virtual void failedMacroDefinitionCheck(unsigned offset, const ByteArrayRef &name) = 0; + virtual void notifyMacroReference(unsigned offset, unsigned line, const Macro ¯o) = 0; + virtual void startExpandingMacro(unsigned offset, + unsigned line, const Macro ¯o, - const ByteArrayRef &originalText, const QVector &actuals = QVector()) = 0; - - virtual void stopExpandingMacro(unsigned offset, - const Macro ¯o) = 0; + virtual void stopExpandingMacro(unsigned offset, const Macro ¯o) = 0; /// Start skipping from the given offset. virtual void startSkippingBlocks(unsigned offset) = 0; virtual void stopSkippingBlocks(unsigned offset) = 0; - virtual void sourceNeeded(QString &fileName, IncludeType mode, - unsigned line) = 0; // ### FIX the signature. + virtual void sourceNeeded(unsigned line, QString &fileName, IncludeType mode) = 0; }; } // namespace CPlusPlus diff --git a/src/libs/cplusplus/PreprocessorEnvironment.h b/src/libs/cplusplus/PreprocessorEnvironment.h index 0e3b4b559ca..a795ee6128b 100644 --- a/src/libs/cplusplus/PreprocessorEnvironment.h +++ b/src/libs/cplusplus/PreprocessorEnvironment.h @@ -95,6 +95,7 @@ private: public: QString currentFile; + QByteArray currentFileUtf8; unsigned currentLine; bool hideNext; diff --git a/src/libs/cplusplus/pp-engine.cpp b/src/libs/cplusplus/pp-engine.cpp index fa780f77fea..ea8c75e0198 100644 --- a/src/libs/cplusplus/pp-engine.cpp +++ b/src/libs/cplusplus/pp-engine.cpp @@ -59,11 +59,13 @@ #include #include -#include -#include #include #include #include +#include + +#include +#include #define NO_DEBUG @@ -75,7 +77,6 @@ namespace { enum { - eagerExpansion = 1, MAX_TOKEN_EXPANSION_COUNT = 5000, MAX_TOKEN_BUFFER_DEPTH = 16000 // for when macros are using some kind of right-folding, this is the list of "delayed" buffers waiting to be expanded after the current one. }; @@ -207,17 +208,21 @@ using namespace CPlusPlus::Internal; namespace { -inline bool isValidToken(const PPToken &tk) +inline bool isContinuationToken(const PPToken &tk) { return tk.isNot(T_EOF_SYMBOL) && (! tk.newline() || tk.joined()); } -Macro *macroDefinition(const ByteArrayRef &name, unsigned offset, Environment *env, Client *client) +Macro *macroDefinition(const ByteArrayRef &name, + unsigned offset, + unsigned line, + Environment *env, + Client *client) { Macro *m = env->resolve(name); if (client) { if (m) - client->passedMacroDefinitionCheck(offset, *m); + client->passedMacroDefinitionCheck(offset, line, *m); else client->failedMacroDefinitionCheck(offset, name); } @@ -352,12 +357,19 @@ protected: } else if (isTokenDefined()) { ++(*_lex); if ((*_lex)->is(T_IDENTIFIER)) { - _value.set_long(macroDefinition(tokenSpell(), (*_lex)->offset, env, client) != 0); + _value.set_long(macroDefinition(tokenSpell(), + (*_lex)->offset, + (*_lex)->lineno, env, client) + != 0); ++(*_lex); } else if ((*_lex)->is(T_LPAREN)) { ++(*_lex); if ((*_lex)->is(T_IDENTIFIER)) { - _value.set_long(macroDefinition(tokenSpell(), (*_lex)->offset, env, client) != 0); + _value.set_long(macroDefinition(tokenSpell(), + (*_lex)->offset, + (*_lex)->lineno, + env, client) + != 0); ++(*_lex); if ((*_lex)->is(T_RPAREN)) { ++(*_lex); @@ -534,15 +546,17 @@ Preprocessor::State::State() , m_tokenBufferDepth(0) , m_inPreprocessorDirective(false) , m_result(0) - , m_markGeneratedTokens(true) + , m_markExpandedTokens(true) , m_noLines(false) , m_inCondition(false) - , m_inDefine(false) , m_offsetRef(0) - , m_envLineRef(1) + , m_lineRef(1) + , m_expansionStatus(NotExpanding) { m_skipping[m_ifLevel] = false; m_trueTest[m_ifLevel] = false; + + m_expansionResult.reserve(256); } //#define COMPRESS_TOKEN_BUFFER @@ -596,8 +610,9 @@ QByteArray Preprocessor::run(const QString &fileName, bool noLines, bool markGeneratedTokens) { + m_scratchBuffer.clear(); + QByteArray preprocessed; -// qDebug()<<"running" << fileName<<"with"<currentFileUtf8); + marker.append("\"\n"); + writeOutput(marker); } void Preprocessor::handleDefined(PPToken *tk) @@ -653,7 +671,7 @@ void Preprocessor::handleDefined(PPToken *tk) idToken = generateConcatenated(idToken, *tk); else break; - } while (isValidToken(*tk)); + } while (isContinuationToken(*tk)); if (lparenSeen && tk->is(T_RPAREN)) @@ -688,32 +706,26 @@ _Lagain: m_state.m_lexer->scan(tk); } + // Adjust token's line number in order to take into account the environment reference. + tk->lineno += m_state.m_lineRef - 1; + _Lclassify: if (! m_state.m_inPreprocessorDirective) { - // Bellow, during directive and identifier handling the current environment line is - // updated in accordance to "global" context in order for clients to rely on consistent - // information. Afterwards, it's restored until output is eventually processed. if (tk->newline() && tk->is(T_POUND)) { - unsigned envLine = m_env->currentLine; - m_env->currentLine = tk->lineno + m_state.m_envLineRef - 1; handlePreprocessorDirective(tk); - m_env->currentLine = envLine; goto _Lclassify; } else if (tk->newline() && skipping()) { ScopedBoolSwap s(m_state.m_inPreprocessorDirective, true); do { lex(tk); - } while (isValidToken(*tk)); + } while (isContinuationToken(*tk)); goto _Lclassify; } else if (tk->is(T_IDENTIFIER) && !isQtReservedWord(tk->asByteArrayRef())) { if (m_state.m_inCondition && tk->asByteArrayRef() == "defined") { handleDefined(tk); } else { - unsigned envLine = m_env->currentLine; - m_env->currentLine = tk->lineno + m_state.m_envLineRef - 1; - bool willExpand = handleIdentifier(tk); - m_env->currentLine = envLine; - if (willExpand) + synchronizeOutputLines(*tk); + if (handleIdentifier(tk)) goto _Lagain; } } @@ -724,7 +736,7 @@ void Preprocessor::skipPreprocesorDirective(PPToken *tk) { ScopedBoolSwap s(m_state.m_inPreprocessorDirective, true); - while (isValidToken(*tk)) { + while (isContinuationToken(*tk)) { lex(tk); } } @@ -742,9 +754,10 @@ bool Preprocessor::handleIdentifier(PPToken *tk) static const QByteArray ppTime("__TIME__"); ByteArrayRef macroNameRef = tk->asByteArrayRef(); - bool newline = tk->newline(); - if (!m_state.m_inDefine && macroNameRef.size() == 8 && macroNameRef[0] == '_' && macroNameRef[1] == '_') { + if (macroNameRef.size() == 8 + && macroNameRef[0] == '_' + && macroNameRef[1] == '_') { PPToken newTk; if (macroNameRef == ppLine) { QByteArray txt = QByteArray::number(tk->lineno); @@ -752,7 +765,7 @@ bool Preprocessor::handleIdentifier(PPToken *tk) } else if (macroNameRef == ppFile) { QByteArray txt; txt.append('"'); - txt.append(m_env->currentFile.toUtf8()); + txt.append(m_env->currentFileUtf8); txt.append('"'); newTk = generateToken(T_STRING_LITERAL, txt.constData(), txt.size(), tk->lineno, false); } else if (macroNameRef == ppDate) { @@ -769,8 +782,8 @@ bool Preprocessor::handleIdentifier(PPToken *tk) newTk = generateToken(T_STRING_LITERAL, txt.constData(), txt.size(), tk->lineno, false); } - if (newTk.isValid()) { - newTk.f.newline = newline; + if (newTk.hasSource()) { + newTk.f.newline = tk->newline(); newTk.f.whitespace = tk->whitespace(); *tk = newTk; return false; @@ -779,22 +792,44 @@ bool Preprocessor::handleIdentifier(PPToken *tk) Macro *macro = m_env->resolve(macroNameRef); if (!macro - || (tk->generated() + || (tk->expanded() && m_state.m_tokenBuffer && m_state.m_tokenBuffer->isBlocked(macro))) { return false; } // qDebug() << "expanding" << macro->name() << "on line" << tk->lineno; - + // Keep track the of the macro identifier token. PPToken idTk = *tk; + + // Expanded tokens which are not generated ones preserve the original line number from + // their corresponding argument in macro substitution. For expanded tokens which are + // generated, this information must be taken from somewhere else. What we do is to keep + // a "reference" line initialize set to the line where expansion happens. + unsigned baseLine = idTk.lineno; + QVector body = macro->definitionTokens(); + // Withing nested expansion we might reach a previously added marker token. In this case, + // we need to move it from its current possition to outside the nesting. + PPToken oldMarkerTk; + if (macro->isFunctionLike()) { // Collect individual tokens that form the macro arguments. QVector > allArgTks; bool hasArgs = collectActualArguments(tk, &allArgTks); + // Check whether collecting arguments failed due to a previously added marker + // that goot nested in a sequence of expansions. If so, store it and try again. + if (!hasArgs + && !tk->hasSource() + && m_state.m_markExpandedTokens + && (m_state.m_expansionStatus == Expanding + || m_state.m_expansionStatus == ReadyForExpansion)) { + oldMarkerTk = *tk; + hasArgs = collectActualArguments(tk, &allArgTks); + } + // Check for matching parameter/argument count. bool hasMatchingArgs = false; if (hasArgs) { @@ -806,13 +841,16 @@ bool Preprocessor::handleIdentifier(PPToken *tk) || (expectedArgCount == 0 && actualArgCount == 1 && allArgTks.at(0).isEmpty())) { - hasMatchingArgs = true; + hasMatchingArgs = true; } } if (!hasArgs || !hasMatchingArgs) { //### TODO: error message pushToken(tk); + // If a previous marker was found, make sure to put it back. + if (oldMarkerTk.f.length) + pushToken(&oldMarkerTk); *tk = idTk; return false; } @@ -835,33 +873,73 @@ bool Preprocessor::handleIdentifier(PPToken *tk) } } - m_client->startExpandingMacro(m_state.m_offsetRef + idTk.offset, *macro, macroNameRef, + m_client->startExpandingMacro(m_state.m_offsetRef + idTk.offset, + idTk.lineno, + *macro, argRefs); } - if (!handleFunctionLikeMacro(tk, macro, body, !m_state.m_inDefine, allArgTks)) { - if (m_client && !idTk.generated()) + if (!handleFunctionLikeMacro(tk, macro, body, allArgTks, baseLine)) { + if (m_client && !idTk.expanded()) m_client->stopExpandingMacro(idTk.offset, *macro); return false; } } else if (m_client && !idTk.generated()) { - m_client->startExpandingMacro(m_state.m_offsetRef + idTk.offset, *macro, macroNameRef); + m_client->startExpandingMacro(m_state.m_offsetRef + idTk.offset, idTk.lineno, *macro); } if (body.isEmpty()) { - if (!m_state.m_inDefine) { - // macro expanded to empty, so characters disappeared, hence force a re-indent. - PPToken forceWhitespacingToken; - // special case: for a macro that expanded to empty, we do not want - // to generate a new #line and re-indent, but just generate the - // amount of spaces that the macro name took up. - forceWhitespacingToken.f.length = tk->length() + (tk->whitespace() ? 1 : 0); - body.push_front(forceWhitespacingToken); + if (m_state.m_markExpandedTokens + && (m_state.m_expansionStatus == NotExpanding + || m_state.m_expansionStatus == JustFinishedExpansion)) { + // This is not the most beautiful approach but it's quite reasonable. What we do here + // is to create a fake identifier token which is only composed by whitespaces. It's + // also not marked as expanded so it it can be treated as a regular token. + QByteArray content(idTk.f.length + computeDistance(idTk), ' '); + PPToken fakeIdentifier = generateToken(T_IDENTIFIER, + content.constData(), content.length(), + idTk.lineno, false, false); + fakeIdentifier.f.whitespace = true; + fakeIdentifier.f.expanded = false; + fakeIdentifier.f.generated = false; + body.push_back(fakeIdentifier); } } else { - PPToken &firstNewTk = body.first(); - firstNewTk.f.newline = newline; - firstNewTk.f.whitespace = true; // the macro call is removed, so space the first token correctly. + // The first body token replaces the macro invocation so its whitespace and + // newline info is replicated. + PPToken &bodyTk = body[0]; + bodyTk.f.whitespace = idTk.f.whitespace; + bodyTk.f.newline = idTk.f.newline; + + // Expansions are tracked from a "top-level" basis. This means that each expansion + // section in the output corresponds to a direct use of a macro (either object-like + // or function-like) in the source code and all its recurring expansions - they are + // surrounded by two marker tokens, one at the begin and the other at the end. + // For instance, the following code will generate 3 expansions in total, but the + // output will aggregate the tokens in only 2 expansion sections. + // - The first corresponds to BAR expanding to FOO and then FOO expanding to T o; + // - The second corresponds to FOO expanding to T o; + // + // #define FOO(T, o) T o; + // #define BAR(T, o) FOO(T, o) + // BAR(Test, x) FOO(Test, y) + if (m_state.m_markExpandedTokens) { + if (m_state.m_expansionStatus == NotExpanding + || m_state.m_expansionStatus == JustFinishedExpansion) { + PPToken marker; + marker.f.expanded = true; + marker.f.length = idTk.f.length; + marker.offset = idTk.offset; + marker.lineno = idTk.lineno; + body.prepend(marker); + body.append(marker); + m_state.m_expansionStatus = ReadyForExpansion; + } else if (oldMarkerTk.f.length + && (m_state.m_expansionStatus == ReadyForExpansion + || m_state.m_expansionStatus == Expanding)) { + body.append(oldMarkerTk); + } + } } m_state.pushTokenBuffer(body.begin(), body.end(), macro); @@ -875,60 +953,88 @@ bool Preprocessor::handleIdentifier(PPToken *tk) bool Preprocessor::handleFunctionLikeMacro(PPToken *tk, const Macro *macro, QVector &body, - bool addWhitespaceMarker, - const QVector > &actuals) + const QVector > &actuals, + unsigned baseLine) { QVector expanded; expanded.reserve(MAX_TOKEN_EXPANSION_COUNT); - for (size_t i = 0, bodySize = body.size(); i < bodySize && expanded.size() < MAX_TOKEN_EXPANSION_COUNT; ++i) { - int expandedSize = expanded.size(); - const PPToken &token = body.at(i); - if (token.is(T_IDENTIFIER)) { - const ByteArrayRef id = token.asByteArrayRef(); + const size_t bodySize = body.size(); + for (size_t i = 0; i < bodySize && expanded.size() < MAX_TOKEN_EXPANSION_COUNT; + ++i) { + int expandedSize = expanded.size(); + PPToken bodyTk = body.at(i); + + if (bodyTk.is(T_IDENTIFIER)) { + const ByteArrayRef id = bodyTk.asByteArrayRef(); const QVector &formals = macro->formals(); int j = 0; for (; j < formals.size() && expanded.size() < MAX_TOKEN_EXPANSION_COUNT; ++j) { if (formals[j] == id) { QVector actualsForThisParam = actuals.at(j); + unsigned lineno = baseLine; + + // Collect variadic arguments if (id == "__VA_ARGS__" || (macro->isVariadic() && j + 1 == formals.size())) { - unsigned lineno = 0; - const char comma = ','; for (int k = j + 1; k < actuals.size(); ++k) { - if (!actualsForThisParam.isEmpty()) - lineno = actualsForThisParam.last().lineno; - actualsForThisParam.append(generateToken(T_COMMA, &comma, 1, lineno, true)); - actualsForThisParam += actuals[k]; + actualsForThisParam.append(generateToken(T_COMMA, ",", 1, lineno, true)); + actualsForThisParam += actuals.at(k); } } - if (i > 0 && body[i - 1].is(T_POUND)) { - QByteArray newText; - newText.reserve(256); - unsigned lineno = 0; - for (int i = 0, ei = actualsForThisParam.size(); i < ei; ++i) { + const int actualsSize = actualsForThisParam.size(); + + if (i > 0 && body[i - 1].is(T_POUND)) { + QByteArray enclosedString; + enclosedString.reserve(256); + + for (int i = 0; i < actualsSize; ++i) { const PPToken &t = actualsForThisParam.at(i); if (i == 0) lineno = t.lineno; else if (t.whitespace()) - newText.append(' '); - newText.append(t.tokenStart(), t.length()); + enclosedString.append(' '); + enclosedString.append(t.tokenStart(), t.length()); + } + enclosedString.replace("\\", "\\\\"); + enclosedString.replace("\"", "\\\""); + + expanded.push_back(generateToken(T_STRING_LITERAL, + enclosedString.constData(), + enclosedString.size(), + lineno, true)); + } else { + for (int k = 0; k < actualsSize; ++k) { + // Mark the actual tokens (which are the replaced version of the + // body's one) as expanded. For the first token we replicate the + // body's whitespace info. + PPToken actual = actualsForThisParam.at(k); + actual.f.expanded = true; + if (k == 0) + actual.f.whitespace = bodyTk.whitespace(); + expanded += actual; + if (k == actualsSize - 1) + lineno = actual.lineno; } - newText.replace("\\", "\\\\"); - newText.replace("\"", "\\\""); - expanded.push_back(generateToken(T_STRING_LITERAL, newText.constData(), newText.size(), lineno, true)); - } else { - for (int k = 0, kk = actualsForThisParam.size(); k < kk; ++k) - expanded += actualsForThisParam.at(k); } + + // Get a better (more up-to-date) value for the base line. + baseLine = lineno; + break; } } - if (j == formals.size()) - expanded.push_back(token); - } else if (token.isNot(T_POUND) && token.isNot(T_POUND_POUND)) { - expanded.push_back(token); + if (j == formals.size()) { + // No formal macro parameter for this identifier in the body. + bodyTk.f.generated = true; + bodyTk.lineno = baseLine; + expanded.push_back(bodyTk); + } + } else if (bodyTk.isNot(T_POUND) && bodyTk.isNot(T_POUND_POUND)) { + bodyTk.f.generated = true; + bodyTk.lineno = baseLine; + expanded.push_back(bodyTk); } if (i > 1 && body[i - 1].is(T_POUND_POUND)) { @@ -941,27 +1047,187 @@ bool Preprocessor::handleFunctionLikeMacro(PPToken *tk, } } - pushToken(tk); - if (addWhitespaceMarker) { - PPToken forceWhitespacingToken; - expanded.push_front(forceWhitespacingToken); - } + // The "new" body. body = expanded; body.squeeze(); + + // Next token to be lexed after the expansion. + pushToken(tk); + return true; } +void Preprocessor::trackExpansionCycles(PPToken *tk) +{ + if (m_state.m_markExpandedTokens) { + // Identify a macro expansion section. The format is as follows: + // + // # expansion begin x,y ~g l:c + // ... + // # expansion end + // + // The x and y correspond, respectively, to the offset where the macro invocation happens + // and the macro name's length. Following that there might be an unlimited number of + // token marks which are directly mapped to each token that appears in the expansion. + // Something like ~g indicates that the following g tokens are all generated. While + // something like l:c indicates that the following token is expanded but not generated + // and is positioned on line l and column c. Example: + // + // #define FOO(X) int f(X = 0) // line 1 + // FOO(int + // a); + // + // Output would be: + // # expansion begin 8,3 ~3 2:4 3:4 ~3 + // int f(int a = 0) + // # expansion end + // # 3 filename + // ; + if (tk->expanded() && !tk->hasSource()) { + if (m_state.m_expansionStatus == ReadyForExpansion) { + m_state.m_expansionStatus = Expanding; + m_state.m_expansionResult.clear(); + m_state.m_expandedTokensInfo.clear(); + } else if (m_state.m_expansionStatus == Expanding) { + m_state.m_expansionStatus = JustFinishedExpansion; + maybeStartOutputLine(); + writeOutput("# expansion begin "); + + QByteArray expansionInfo; + expansionInfo.reserve(m_state.m_expandedTokensInfo.size() * 2); // Rough estimate + + // Offset and length of the macro invocation + expansionInfo.append(QByteArray::number(tk->offset)); + expansionInfo.append(','); + expansionInfo.append(QByteArray::number(tk->length())); + + // Expanded tokens + unsigned generatedCount = 0; + for (int i = 0; i < m_state.m_expandedTokensInfo.size(); ++i) { + const QPair &p = m_state.m_expandedTokensInfo.at(i); + if (p.first) { + if (generatedCount) { + expansionInfo.append(" ~"); + expansionInfo.append(QByteArray::number(generatedCount)); + generatedCount = 0; + } + expansionInfo.append(' '); + expansionInfo.append(QByteArray::number(p.first)); + expansionInfo.append(':'); + expansionInfo.append(QByteArray::number(p.second)); + } else { + ++generatedCount; + } + } + if (generatedCount) { + expansionInfo.append(" ~"); + expansionInfo.append(QByteArray::number(generatedCount)); + } + expansionInfo.append('\n'); + + writeOutput(expansionInfo); + writeOutput(m_state.m_expansionResult); + maybeStartOutputLine(); + writeOutput("# expansion end\n"); + } + + lex(tk); + + if (tk->expanded() && !tk->hasSource()) + trackExpansionCycles(tk); + } + } +} + +void Preprocessor::synchronizeOutputLines(const PPToken &tk, bool forceLine) +{ + if (m_state.m_expansionStatus != NotExpanding + || (!forceLine && m_env->currentLine == tk.lineno)) { + return; + } + + if (forceLine || m_env->currentLine > tk.lineno || tk.lineno - m_env->currentLine >= 9) { + if (m_state.m_noLines) { + if (!m_state.m_markExpandedTokens) + writeOutput(' '); + } else { + generateOutputLineMarker(tk.lineno); + } + } else { + for (unsigned i = m_env->currentLine; i < tk.lineno; ++i) + writeOutput('\n'); + } + + m_env->currentLine = tk.lineno; + if (tk.is(T_COMMENT) || tk.is(T_DOXY_COMMENT)) + m_env->currentLine += tk.asByteArrayRef().count('\n'); +} + +void Preprocessor::removeTrailingOutputLines() +{ + QByteArray *buffer = currentOutputBuffer(); + if (buffer) { + int i = buffer->size() - 1; + while (i >= 0 && buffer->at(i) == '\n') + --i; + const int mightChop = buffer->size() - i - 1; + if (mightChop > 1) { + // Keep one new line at end. + buffer->chop(mightChop - 1); + } + } +} + +std::size_t Preprocessor::computeDistance(const Preprocessor::PPToken &tk, bool forceTillLine) +{ + // Find previous non-space character or line begin. + const char *buffer = tk.bufferStart(); + const char *tokenBegin = tk.tokenStart(); + const char *it = tokenBegin - 1; + for (; it >= buffer; --it) { + if (*it == '\n'|| (!pp_isspace(*it) && !forceTillLine)) + break; + } + ++it; + + return tokenBegin - it; +} + + +void Preprocessor::enforceSpacing(const Preprocessor::PPToken &tk, bool forceSpacing) +{ + if (tk.whitespace() || forceSpacing) { + // For expanded tokens we simply add a whitespace, if necessary - the exact amount of + // whitespaces is irrelevant within an expansion section. For real tokens we must be + // more specific and get the information from the original source. + if (tk.expanded() && !atStartOfOutputLine()) { + writeOutput(' '); + } else { + const std::size_t spacing = computeDistance(tk, forceSpacing); + const char *tokenBegin = tk.tokenStart(); + const char *it = tokenBegin - spacing; + + // Reproduce the content as in the original line. + for (; it != tokenBegin; ++it) { + if (pp_isspace(*it)) + writeOutput(*it); + else + writeOutput(' '); + } + } + } +} + /// invalid pp-tokens are used as markers to force whitespace checks. void Preprocessor::preprocess(const QString &fileName, const QByteArray &source, QByteArray *result, bool noLines, bool markGeneratedTokens, bool inCondition, - unsigned offsetRef, unsigned envLineRef) + unsigned offsetRef, unsigned lineRef) { if (source.isEmpty()) return; const State savedState = m_state; - m_state = State(); m_state.m_currentFileName = fileName; m_state.m_source = source; @@ -972,103 +1238,62 @@ void Preprocessor::preprocess(const QString &fileName, const QByteArray &source, m_state.m_lexer->setScanCommentTokens(true); m_state.m_result = result; m_state.m_noLines = noLines; - m_state.m_markGeneratedTokens = markGeneratedTokens; + m_state.m_markExpandedTokens = markGeneratedTokens; m_state.m_inCondition = inCondition; m_state.m_offsetRef = offsetRef; - m_state.m_envLineRef = envLineRef; + m_state.m_lineRef = lineRef; const QString previousFileName = m_env->currentFile; m_env->currentFile = fileName; + m_env->currentFileUtf8 = fileName.toUtf8(); const unsigned previousCurrentLine = m_env->currentLine; m_env->currentLine = 1; - const QByteArray fn = fileName.toUtf8(); if (!m_state.m_noLines) - genLine(1, fn); + generateOutputLineMarker(1); - PPToken tk(m_state.m_source), prevTk; + PPToken tk(m_state.m_source); do { -_Lrestart: - bool forceLine = false; lex(&tk); - if (!tk.isValid()) { - bool wasGenerated = prevTk.generated(); - prevTk = tk; - prevTk.f.generated = wasGenerated; - goto _Lrestart; - } + // Track the start and end of macro expansion cycles. + trackExpansionCycles(&tk); - if (m_state.m_markGeneratedTokens && tk.generated() && !prevTk.generated()) { - startNewOutputLine(); - out("#gen true\n"); - ++m_env->currentLine; - forceLine = true; - } else if (m_state.m_markGeneratedTokens && !tk.generated() && prevTk.generated()) { - startNewOutputLine(); - out("#gen false\n"); - ++m_env->currentLine; - forceLine = true; - } - - if (forceLine || m_env->currentLine != tk.lineno) { - if (forceLine || m_env->currentLine > tk.lineno || tk.lineno - m_env->currentLine > 3) { - if (m_state.m_noLines) { - if (!m_state.m_markGeneratedTokens) - out(' '); - } else { - genLine(tk.lineno, fn); - } - } else { - for (unsigned i = m_env->currentLine; i < tk.lineno; ++i) - out('\n'); + bool macroExpanded = false; + if (m_state.m_expansionStatus == Expanding) { + // Collect the line and column from the tokens undergoing expansion. Those will + // be available in the expansion section for further referencing about their real + // location. + unsigned trackedLine = 0; + unsigned trackedColumn = 0; + if (tk.expanded() && !tk.generated()) { + trackedLine = tk.lineno; + trackedColumn = computeDistance(tk, true); } - } else { - if (tk.newline() && prevTk.isValid()) - out('\n'); + m_state.m_expandedTokensInfo.append(qMakePair(trackedLine, trackedColumn)); + } else if (m_state.m_expansionStatus == JustFinishedExpansion) { + m_state.m_expansionStatus = NotExpanding; + macroExpanded = true; } - if (tk.whitespace() || prevTk.generated() != tk.generated() || !prevTk.isValid()) { - if (prevTk.generated() && tk.generated()) { - out(' '); - } else if (tk.isValid() && !prevTk.isValid() && tk.lineno == m_env->currentLine) { - out(QByteArray(prevTk.length() + (tk.whitespace() ? 1 : 0), ' ')); - } else if (prevTk.generated() != tk.generated() || !prevTk.isValid()) { - const char *begin = tk.bufferStart(); - const char *end = tk.tokenStart(); - const char *it = end - 1; - for (; it >= begin; --it) - if (*it == '\n') - break; - ++it; - for (; it < end; ++it) - out(' '); - } else { - const char *begin = tk.bufferStart(); - const char *end = tk.tokenStart(); - const char *it = end - 1; - for (; it >= begin; --it) - if (!pp_isspace(*it) || *it == '\n') - break; - ++it; - for (; it < end; ++it) - out(*it); - } - } + // Update environment line information. + synchronizeOutputLines(tk, macroExpanded); - const ByteArrayRef tkBytes = tk.asByteArrayRef(); - out(tkBytes); - m_env->currentLine = tk.lineno; - if (tk.is(T_COMMENT) || tk.is(T_DOXY_COMMENT)) - m_env->currentLine += tkBytes.count('\n'); - prevTk = tk; + // Make sure spacing between tokens is handled properly. + enforceSpacing(tk, macroExpanded); + + // Finally output the token. + writeOutput(tk.asByteArrayRef()); } while (tk.isNot(T_EOF_SYMBOL)); + removeTrailingOutputLines(); + delete m_state.m_lexer; m_state = savedState; m_env->currentFile = previousFileName; + m_env->currentFileUtf8 = previousFileName.toUtf8(); m_env->currentLine = previousCurrentLine; } @@ -1212,7 +1437,7 @@ void Preprocessor::handleIncludeDirective(PPToken *tk) if (m_client) { QString inc = QString::fromUtf8(included.constData() + 1, included.size() - 2); - m_client->sourceNeeded(inc, mode, line); + m_client->sourceNeeded(line, inc, mode); } } @@ -1221,12 +1446,9 @@ void Preprocessor::handleDefineDirective(PPToken *tk) const unsigned defineOffset = tk->offset; lex(tk); // consume "define" token - bool hasIdentifier = false; if (tk->isNot(T_IDENTIFIER)) return; - ScopedBoolSwap inDefine(m_state.m_inDefine, true); - Macro macro; macro.setFileName(m_env->currentFile); macro.setLine(tk->lineno); @@ -1236,21 +1458,22 @@ void Preprocessor::handleDefineDirective(PPToken *tk) lex(tk); - if (isValidToken(*tk) && tk->is(T_LPAREN) && ! tk->whitespace()) { + if (isContinuationToken(*tk) && tk->is(T_LPAREN) && ! tk->whitespace()) { macro.setFunctionLike(true); lex(tk); // skip `(' - if (isValidToken(*tk) && tk->is(T_IDENTIFIER)) { + bool hasIdentifier = false; + if (isContinuationToken(*tk) && tk->is(T_IDENTIFIER)) { hasIdentifier = true; macro.addFormal(tk->asByteArrayRef().toByteArray()); lex(tk); - while (isValidToken(*tk) && tk->is(T_COMMA)) { + while (isContinuationToken(*tk) && tk->is(T_COMMA)) { lex(tk); - if (isValidToken(*tk) && tk->is(T_IDENTIFIER)) { + if (isContinuationToken(*tk) && tk->is(T_IDENTIFIER)) { macro.addFormal(tk->asByteArrayRef().toByteArray()); lex(tk); } else { @@ -1265,28 +1488,44 @@ void Preprocessor::handleDefineDirective(PPToken *tk) macro.addFormal("__VA_ARGS__"); lex(tk); // consume elipsis token } - if (isValidToken(*tk) && tk->is(T_RPAREN)) + if (isContinuationToken(*tk) && tk->is(T_RPAREN)) lex(tk); // consume ")" token } QVector bodyTokens; - PPToken firstBodyToken = *tk; - while (isValidToken(*tk)) { - if (eagerExpansion) { - PPToken idTk = *tk; - while (tk->is(T_IDENTIFIER) - && (!tk->newline() || tk->joined()) - && !isQtReservedWord(tk->asByteArrayRef()) - && handleIdentifier(tk)) { - lex(tk); - if (tk->asByteArrayRef() == macroName) { - *tk = idTk; - break; + unsigned previousOffset = 0; + unsigned previousLine = 0; + Macro *macroReference = 0; + while (isContinuationToken(*tk)) { + // Macro tokens are always marked as expanded. However, only for object-like macros + // we mark them as generated too. For function-like macros we postpone it until the + // formals are identified in the bodies. + tk->f.expanded = true; + if (!macro.isFunctionLike()) + tk->f.generated = true; + + // Identifiers must not be eagerly expanded inside defines, but we should still notify + // in the case they are macros. + if (tk->is(T_IDENTIFIER) && m_client) { + macroReference = m_env->resolve(tk->asByteArrayRef()); + if (macroReference) { + if (!macroReference->isFunctionLike()) { + m_client->notifyMacroReference(tk->offset, tk->lineno, *macroReference); + macroReference = 0; } } + } else if (macroReference) { + if (tk->is(T_LPAREN)) { + m_client->notifyMacroReference(previousOffset, previousLine, *macroReference); + } + macroReference = 0; } - tk->f.generated = true; + + previousOffset = tk->offset; + previousLine = tk->lineno; + bodyTokens.push_back(*tk); + lex(tk); } @@ -1307,14 +1546,17 @@ void Preprocessor::handleDefineDirective(PPToken *tk) bodyTokens.clear(); macro.setDefinition(macroId, bodyTokens); - } else { + } else if (!bodyTokens.isEmpty()) { + PPToken &firstBodyToken = bodyTokens[0]; int start = firstBodyToken.offset; int len = tk->offset - start; QByteArray bodyText = firstBodyToken.source().mid(start, len).trimmed(); - for (int i = 0, count = bodyTokens.size(); i < count; ++i) { + + const int bodySize = bodyTokens.size(); + for (int i = 0; i < bodySize; ++i) { PPToken &t = bodyTokens[i]; - if (t.isValid()) - t.squeeze(); + if (t.hasSource()) + t.squeezeSource(); } macro.setDefinition(bodyText, bodyTokens); } @@ -1330,9 +1572,10 @@ void Preprocessor::handleDefineDirective(PPToken *tk) QByteArray Preprocessor::expand(PPToken *tk, PPToken *lastConditionToken) { + unsigned line = tk->lineno; unsigned begin = tk->begin(); PPToken lastTk; - while (isValidToken(*tk)) { + while (isContinuationToken(*tk)) { lastTk = *tk; lex(tk); } @@ -1342,8 +1585,7 @@ QByteArray Preprocessor::expand(PPToken *tk, PPToken *lastConditionToken) // qDebug("*** Condition before: [%s]", condition.constData()); QByteArray result; result.reserve(256); - preprocess(m_state.m_currentFileName, condition, &result, true, false, true, begin, - m_env->currentLine); + preprocess(m_state.m_currentFileName, condition, &result, true, false, true, begin, line); result.squeeze(); // qDebug("*** Condition after: [%s]", result.constData()); @@ -1477,7 +1719,7 @@ void Preprocessor::handleIfDefDirective(bool checkUndefined, PPToken *tk) if (tk->is(T_IDENTIFIER)) { bool value = false; const ByteArrayRef macroName = tk->asByteArrayRef(); - if (Macro *macro = macroDefinition(macroName, tk->offset, m_env, m_client)) { + if (Macro *macro = macroDefinition(macroName, tk->offset, tk->lineno, m_env, m_client)) { value = true; // the macro is a feature constraint(e.g. QT_NO_XXX) @@ -1574,10 +1816,13 @@ bool Preprocessor::isQtReservedWord(const ByteArrayRef ¯oId) return false; } -PPToken Preprocessor::generateToken(enum Kind kind, const char *content, int len, unsigned lineno, bool addQuotes) + +PPToken Preprocessor::generateToken(enum Kind kind, + const char *content, int length, + unsigned lineno, + bool addQuotes, + bool addToControl) { - // When reconstructing the column position of a token, - // Preprocessor::preprocess will search for the last preceeding newline. // When the token is a generated token, the column position cannot be // reconstructed, but we also have to prevent it from searching the whole // scratch buffer. So inserting a newline before the new token will give @@ -1588,27 +1833,29 @@ PPToken Preprocessor::generateToken(enum Kind kind, const char *content, int len if (kind == T_STRING_LITERAL && addQuotes) m_scratchBuffer.append('"'); - m_scratchBuffer.append(content, len); + m_scratchBuffer.append(content, length); if (kind == T_STRING_LITERAL && addQuotes) { m_scratchBuffer.append('"'); - len += 2; + length += 2; } - PPToken tok(m_scratchBuffer); - tok.f.kind = kind; - if (m_state.m_lexer->control()) { + PPToken tk(m_scratchBuffer); + tk.f.kind = kind; + if (m_state.m_lexer->control() && addToControl) { if (kind == T_STRING_LITERAL) - tok.string = m_state.m_lexer->control()->stringLiteral(m_scratchBuffer.constData() + pos, len); + tk.string = m_state.m_lexer->control()->stringLiteral(m_scratchBuffer.constData() + pos, length); else if (kind == T_IDENTIFIER) - tok.identifier = m_state.m_lexer->control()->identifier(m_scratchBuffer.constData() + pos, len); + tk.identifier = m_state.m_lexer->control()->identifier(m_scratchBuffer.constData() + pos, length); else if (kind == T_NUMERIC_LITERAL) - tok.number = m_state.m_lexer->control()->numericLiteral(m_scratchBuffer.constData() + pos, len); + tk.number = m_state.m_lexer->control()->numericLiteral(m_scratchBuffer.constData() + pos, length); } - tok.offset = pos; - tok.f.length = len; - tok.f.generated = true; - tok.lineno = lineno; - return tok; + tk.offset = pos; + tk.f.length = length; + tk.f.generated = true; + tk.f.expanded = true; + tk.lineno = lineno; + + return tk; } PPToken Preprocessor::generateConcatenated(const PPToken &leftTk, const PPToken &rightTk) @@ -1636,3 +1883,44 @@ void Preprocessor::startSkippingBlocks(const Preprocessor::PPToken &tk) const } } } + +template +void Preprocessor::writeOutput(const T &t) +{ + QByteArray *buffer = currentOutputBuffer(); + if (buffer) + buffer->append(t); +} + +void Preprocessor::writeOutput(const ByteArrayRef &ref) +{ + QByteArray *buffer = currentOutputBuffer(); + if (buffer) + buffer->append(ref.start(), ref.length()); +} + +bool Preprocessor::atStartOfOutputLine() const +{ + const QByteArray *buffer = currentOutputBuffer(); + return (buffer && !buffer->isEmpty()) ? *(buffer->end() - 1) == '\n' : true; +} + +void Preprocessor::maybeStartOutputLine() +{ + QByteArray *buffer = currentOutputBuffer(); + if (buffer && !buffer->isEmpty() && *(buffer->end() - 1) != '\n') + writeOutput('\n'); +} + +const QByteArray *Preprocessor::currentOutputBuffer() const +{ + if (m_state.m_expansionStatus == Expanding) + return &m_state.m_expansionResult; + + return m_state.m_result; +} + +QByteArray *Preprocessor::currentOutputBuffer() +{ + return const_cast(static_cast(this)->currentOutputBuffer()); +} diff --git a/src/libs/cplusplus/pp-engine.h b/src/libs/cplusplus/pp-engine.h index c310a9e3e3a..478812918d3 100644 --- a/src/libs/cplusplus/pp-engine.h +++ b/src/libs/cplusplus/pp-engine.h @@ -60,6 +60,7 @@ #include #include #include +#include namespace CPlusPlus { @@ -92,10 +93,17 @@ private: void preprocess(const QString &filename, const QByteArray &source, QByteArray *result, bool noLines, bool markGeneratedTokens, bool inCondition, - unsigned offsetRef = 0, unsigned envLineRef = 1); + unsigned offsetRef = 0, unsigned lineRef = 1); enum { MAX_LEVEL = 512 }; + enum ExpansionStatus { + NotExpanding, + ReadyForExpansion, + Expanding, + JustFinishedExpansion + }; + struct State { State(); @@ -114,14 +122,17 @@ private: bool m_inPreprocessorDirective; QByteArray *m_result; - bool m_markGeneratedTokens; + bool m_markExpandedTokens; bool m_noLines; bool m_inCondition; - bool m_inDefine; unsigned m_offsetRef; - unsigned m_envLineRef; + unsigned m_lineRef; + + ExpansionStatus m_expansionStatus; + QByteArray m_expansionResult; + QVector > m_expandedTokensInfo; }; void handleDefined(PPToken *tk); @@ -129,9 +140,11 @@ private: void lex(PPToken *tk); void skipPreprocesorDirective(PPToken *tk); bool handleIdentifier(PPToken *tk); - bool handleFunctionLikeMacro(PPToken *tk, const Macro *macro, QVector &body, - bool addWhitespaceMarker, - const QVector > &actuals); + bool handleFunctionLikeMacro(PPToken *tk, + const Macro *macro, + QVector &body, + const QVector > &actuals, + unsigned lineRef); bool skipping() const { return m_state.m_skipping[m_state.m_ifLevel]; } @@ -155,30 +168,28 @@ private: static bool isQtReservedWord(const ByteArrayRef &name); - inline bool atStartOfOutputLine() const - { return (m_state.m_result && !m_state.m_result->isEmpty()) ? m_state.m_result->end()[-1] == '\n' : true; } + void trackExpansionCycles(PPToken *tk); - inline void startNewOutputLine() const - { - if (m_state.m_result && !m_state.m_result->isEmpty() && m_state.m_result->end()[-1] != '\n') - out('\n'); - } + template + void writeOutput(const T &t); + void writeOutput(const ByteArrayRef &ref); + bool atStartOfOutputLine() const; + void maybeStartOutputLine(); + void generateOutputLineMarker(unsigned lineno); + void synchronizeOutputLines(const PPToken &tk, bool forceLine = false); + void removeTrailingOutputLines(); - void genLine(unsigned lineno, const QByteArray &fileName) const; + const QByteArray *currentOutputBuffer() const; + QByteArray *currentOutputBuffer(); - inline void out(const QByteArray &text) const - { if (m_state.m_result) m_state.m_result->append(text); } + void enforceSpacing(const PPToken &tk, bool forceSpacing = false); + static std::size_t computeDistance(const PPToken &tk, bool forceTillLine = false); - inline void out(char ch) const - { if (m_state.m_result) m_state.m_result->append(ch); } - - inline void out(const char *s) const - { if (m_state.m_result) m_state.m_result->append(s); } - - inline void out(const ByteArrayRef &ref) const - { if (m_state.m_result) m_state.m_result->append(ref.start(), ref.length()); } - - PPToken generateToken(enum Kind kind, const char *content, int len, unsigned lineno, bool addQuotes); + PPToken generateToken(enum Kind kind, + const char *content, int length, + unsigned lineno, + bool addQuotes, + bool addToControl = true); PPToken generateConcatenated(const PPToken &leftTk, const PPToken &rightTk); void startSkippingBlocks(const PPToken &tk) const; diff --git a/src/plugins/cpptools/cppmodelmanager.cpp b/src/plugins/cpptools/cppmodelmanager.cpp index 31e8ca3e7ca..53035db575c 100644 --- a/src/plugins/cpptools/cppmodelmanager.cpp +++ b/src/plugins/cpptools/cppmodelmanager.cpp @@ -340,7 +340,7 @@ public: void CppPreprocessor::run(const QString &fileName) { QString absoluteFilePath = fileName; - sourceNeeded(absoluteFilePath, IncludeGlobal, /*line = */ 0); + sourceNeeded(0, absoluteFilePath, IncludeGlobal); } void CppPreprocessor::resetEnvironment() @@ -499,12 +499,12 @@ void CppPreprocessor::macroAdded(const Macro ¯o) m_currentDoc->appendMacro(macro); } -void CppPreprocessor::passedMacroDefinitionCheck(unsigned offset, const Macro ¯o) +void CppPreprocessor::passedMacroDefinitionCheck(unsigned offset, unsigned line, const Macro ¯o) { if (! m_currentDoc) return; - m_currentDoc->addMacroUse(macro, offset, macro.name().length(), env.currentLine, + m_currentDoc->addMacroUse(macro, offset, macro.name().length(), line, QVector()); } @@ -516,15 +516,23 @@ void CppPreprocessor::failedMacroDefinitionCheck(unsigned offset, const ByteArra m_currentDoc->addUndefinedMacroUse(QByteArray(name.start(), name.size()), offset); } -void CppPreprocessor::startExpandingMacro(unsigned offset, +void CppPreprocessor::notifyMacroReference(unsigned offset, unsigned line, const Macro ¯o) +{ + if (! m_currentDoc) + return; + + m_currentDoc->addMacroUse(macro, offset, macro.name().length(), line, + QVector()); +} + +void CppPreprocessor::startExpandingMacro(unsigned offset, unsigned line, const Macro ¯o, - const ByteArrayRef &originalText, const QVector &actuals) { if (! m_currentDoc) return; - m_currentDoc->addMacroUse(macro, offset, originalText.length(), env.currentLine, actuals); + m_currentDoc->addMacroUse(macro, offset, macro.name().length(), line, actuals); } void CppPreprocessor::stopExpandingMacro(unsigned, const Macro &) @@ -573,7 +581,7 @@ void CppPreprocessor::stopSkippingBlocks(unsigned offset) m_currentDoc->stopSkippingBlocks(offset); } -void CppPreprocessor::sourceNeeded(QString &fileName, IncludeType type, unsigned line) +void CppPreprocessor::sourceNeeded(unsigned line, QString &fileName, IncludeType type) { if (fileName.isEmpty()) return; @@ -590,7 +598,7 @@ void CppPreprocessor::sourceNeeded(QString &fileName, IncludeType type, unsigned Document::DiagnosticMessage d(Document::DiagnosticMessage::Warning, m_currentDoc->fileName(), - env.currentLine, /*column = */ 0, + line, /*column = */ 0, msg); m_currentDoc->addDiagnosticMessage(d); diff --git a/src/plugins/cpptools/cppmodelmanager.h b/src/plugins/cpptools/cppmodelmanager.h index c165667a2ff..863b981c4ed 100644 --- a/src/plugins/cpptools/cppmodelmanager.h +++ b/src/plugins/cpptools/cppmodelmanager.h @@ -300,17 +300,19 @@ protected: void mergeEnvironment(CPlusPlus::Document::Ptr doc); virtual void macroAdded(const CPlusPlus::Macro ¯o); - virtual void passedMacroDefinitionCheck(unsigned offset, const CPlusPlus::Macro ¯o); + virtual void passedMacroDefinitionCheck(unsigned offset, unsigned line, + const CPlusPlus::Macro ¯o); virtual void failedMacroDefinitionCheck(unsigned offset, const CPlusPlus::ByteArrayRef &name); + virtual void notifyMacroReference(unsigned offset, unsigned line, + const CPlusPlus::Macro ¯o); virtual void startExpandingMacro(unsigned offset, + unsigned line, const CPlusPlus::Macro ¯o, - const CPlusPlus::ByteArrayRef &originalText, const QVector &actuals); virtual void stopExpandingMacro(unsigned offset, const CPlusPlus::Macro ¯o); virtual void startSkippingBlocks(unsigned offset); virtual void stopSkippingBlocks(unsigned offset); - virtual void sourceNeeded(QString &fileName, IncludeType type, - unsigned line); + virtual void sourceNeeded(unsigned line, QString &fileName, IncludeType type); private: #ifndef ICHECK_BUILD diff --git a/tests/auto/cplusplus/preprocessor/data/empty-macro.2.out.cpp b/tests/auto/cplusplus/preprocessor/data/empty-macro.2.out.cpp index fced1fa8fd1..03703e9bbd9 100644 --- a/tests/auto/cplusplus/preprocessor/data/empty-macro.2.out.cpp +++ b/tests/auto/cplusplus/preprocessor/data/empty-macro.2.out.cpp @@ -1,32 +1,15 @@ # 1 "data/empty-macro.2.cpp" -# 6 "data/empty-macro.2.cpp" + + + + + class Test { private: - Test -#gen true -# 3 "data/empty-macro.2.cpp" -(const -#gen false -# 8 "data/empty-macro.2.cpp" - Test -#gen true -# 3 "data/empty-macro.2.cpp" -&); -#gen false -# 8 "data/empty-macro.2.cpp" - Test -#gen true -# 4 "data/empty-macro.2.cpp" -&operator=(const -#gen false -# 8 "data/empty-macro.2.cpp" - Test -#gen true -# 4 "data/empty-macro.2.cpp" -&); -#gen false +# expansion begin 182,14 8:19 ~2 8:19 ~3 8:19 ~5 8:19 ~3 +Test(const Test &); Test &operator=(const Test &); +# expansion end # 10 "data/empty-macro.2.cpp" public: Test(); }; - diff --git a/tests/auto/cplusplus/preprocessor/data/empty-macro.cpp b/tests/auto/cplusplus/preprocessor/data/empty-macro.cpp index d6ef56c60e2..b86fccb7b9d 100644 --- a/tests/auto/cplusplus/preprocessor/data/empty-macro.cpp +++ b/tests/auto/cplusplus/preprocessor/data/empty-macro.cpp @@ -3,3 +3,11 @@ class EMPTY_MACRO Foo { }; +class EMPTY_MACRO Foo2 { +}; + +class EMPTY_MACRO Foo3 { +}; + +class EMPTY_MACRO Foo3 { +}; diff --git a/tests/auto/cplusplus/preprocessor/data/empty-macro.out.cpp b/tests/auto/cplusplus/preprocessor/data/empty-macro.out.cpp index 4e3be9cb227..405cbbacf91 100644 --- a/tests/auto/cplusplus/preprocessor/data/empty-macro.out.cpp +++ b/tests/auto/cplusplus/preprocessor/data/empty-macro.out.cpp @@ -4,3 +4,11 @@ class Foo { }; +class Foo2 { +}; + +class Foo3 { +}; + +class Foo3 { +}; diff --git a/tests/auto/cplusplus/preprocessor/data/identifier-expansion.1.out.cpp b/tests/auto/cplusplus/preprocessor/data/identifier-expansion.1.out.cpp index b8e83bb077f..a401d55dbfc 100644 --- a/tests/auto/cplusplus/preprocessor/data/identifier-expansion.1.out.cpp +++ b/tests/auto/cplusplus/preprocessor/data/identifier-expansion.1.out.cpp @@ -1,15 +1,18 @@ # 1 "data/identifier-expansion.1.cpp" -#gen true -# 1 "data/identifier-expansion.1.cpp" -test test -#gen false + + +# expansion begin 19,4 ~1 +test +# expansion end +# expansion begin 24,4 ~1 +test +# expansion end # 3 "data/identifier-expansion.1.cpp" ; void -#gen true -# 1 "data/identifier-expansion.1.cpp" +# expansion begin 36,4 ~1 test -#gen false +# expansion end # 5 "data/identifier-expansion.1.cpp" (); diff --git a/tests/auto/cplusplus/preprocessor/data/identifier-expansion.2.out.cpp b/tests/auto/cplusplus/preprocessor/data/identifier-expansion.2.out.cpp index d6e37e4ac57..8d4c833a0de 100644 --- a/tests/auto/cplusplus/preprocessor/data/identifier-expansion.2.out.cpp +++ b/tests/auto/cplusplus/preprocessor/data/identifier-expansion.2.out.cpp @@ -1,15 +1,19 @@ # 1 "data/identifier-expansion.2.cpp" -#gen true -# 1 "data/identifier-expansion.2.cpp" -test test -#gen false + + + +# expansion begin 45,12 ~1 +test +# expansion end +# expansion begin 58,4 ~1 +test +# expansion end # 4 "data/identifier-expansion.2.cpp" ; void -#gen true -# 1 "data/identifier-expansion.2.cpp" +# expansion begin 70,12 ~1 test -#gen false +# expansion end # 6 "data/identifier-expansion.2.cpp" (); diff --git a/tests/auto/cplusplus/preprocessor/data/identifier-expansion.3.cpp b/tests/auto/cplusplus/preprocessor/data/identifier-expansion.3.cpp index cf85dc953f7..4f0e0bd81c3 100644 --- a/tests/auto/cplusplus/preprocessor/data/identifier-expansion.3.cpp +++ b/tests/auto/cplusplus/preprocessor/data/identifier-expansion.3.cpp @@ -2,13 +2,20 @@ V(ADD) \ V(SUB) +#define OTHER_FOR_EACH(V) \ + V(DIV) \ + V(MUL) + #define DECLARE_INSTR(op) #op, #define DECLARE_OP_INSTR(op) op_##op, enum op_code { FOR_EACH_INSTR(DECLARE_OP_INSTR) + OTHER_FOR_EACH(DECLARE_OP_INSTR) }; + static const char *names[] = { FOR_EACH_INSTR(DECLARE_INSTR) +OTHER_FOR_EACH(DECLARE_INSTR) }; diff --git a/tests/auto/cplusplus/preprocessor/data/identifier-expansion.3.out.cpp b/tests/auto/cplusplus/preprocessor/data/identifier-expansion.3.out.cpp index 689220f32e6..bd8b2bdbf2c 100644 --- a/tests/auto/cplusplus/preprocessor/data/identifier-expansion.3.out.cpp +++ b/tests/auto/cplusplus/preprocessor/data/identifier-expansion.3.out.cpp @@ -1,24 +1,22 @@ # 1 "data/identifier-expansion.3.cpp" -# 8 "data/identifier-expansion.3.cpp" +# 12 "data/identifier-expansion.3.cpp" enum op_code { -#gen true -# 6 "data/identifier-expansion.3.cpp" +# expansion begin 195,14 ~4 op_ADD, op_SUB, -#gen false -# 10 "data/identifier-expansion.3.cpp" +# expansion end +# expansion begin 232,14 ~4 +op_DIV, op_MUL, +# expansion end +# 15 "data/identifier-expansion.3.cpp" }; + static const char *names[] = { -#gen true -# 2 "data/identifier-expansion.3.cpp" -"ADD" - - -, -# 3 "data/identifier-expansion.3.cpp" - "SUB" - -, -#gen false -# 14 "data/identifier-expansion.3.cpp" +# expansion begin 301,14 ~4 +"ADD", "SUB", +# expansion end +# expansion begin 331,14 ~4 +"DIV", "MUL", +# expansion end +# 21 "data/identifier-expansion.3.cpp" }; diff --git a/tests/auto/cplusplus/preprocessor/data/identifier-expansion.4.out.cpp b/tests/auto/cplusplus/preprocessor/data/identifier-expansion.4.out.cpp index e346f408f6e..759eae44186 100644 --- a/tests/auto/cplusplus/preprocessor/data/identifier-expansion.4.out.cpp +++ b/tests/auto/cplusplus/preprocessor/data/identifier-expansion.4.out.cpp @@ -5,5 +5,9 @@ void baz() { int aaa; - aaa; +# expansion begin 88,4 7:9 +aaa +# expansion end +# 7 "data/identifier-expansion.4.cpp" + ; } diff --git a/tests/auto/cplusplus/preprocessor/data/identifier-expansion.5.out.cpp b/tests/auto/cplusplus/preprocessor/data/identifier-expansion.5.out.cpp index 0bab742e80d..3d328fe2513 100644 --- a/tests/auto/cplusplus/preprocessor/data/identifier-expansion.5.out.cpp +++ b/tests/auto/cplusplus/preprocessor/data/identifier-expansion.5.out.cpp @@ -1,2 +1 @@ # 1 "data/identifier-expansion.5.cpp" -# 9 "data/identifier-expansion.5.cpp" diff --git a/tests/auto/cplusplus/preprocessor/data/macro-test.out.cpp b/tests/auto/cplusplus/preprocessor/data/macro-test.out.cpp index cc8737a7416..d13e2c5b0b3 100644 --- a/tests/auto/cplusplus/preprocessor/data/macro-test.out.cpp +++ b/tests/auto/cplusplus/preprocessor/data/macro-test.out.cpp @@ -1,8 +1,12 @@ # 1 "data/macro-test.cpp" -# 7 "data/macro-test.cpp" + + + + + + void thisFunctionIsEnabled(); # 21 "data/macro-test.cpp" void thisFunctionIsEnabled2(); # 31 "data/macro-test.cpp" void thisFunctionIsEnabled3(); -# 37 "data/macro-test.cpp" diff --git a/tests/auto/cplusplus/preprocessor/data/macro_expand.out.c b/tests/auto/cplusplus/preprocessor/data/macro_expand.out.c index 918ad379fff..24442a7567c 100644 --- a/tests/auto/cplusplus/preprocessor/data/macro_expand.out.c +++ b/tests/auto/cplusplus/preprocessor/data/macro_expand.out.c @@ -3,8 +3,7 @@ A: -#gen true -# 1 "data/macro_expand.c" +# expansion begin 32,1 ~1 Y -#gen false +# expansion end # 5 "data/macro_expand.c" diff --git a/tests/auto/cplusplus/preprocessor/data/macro_expand_1.out.cpp b/tests/auto/cplusplus/preprocessor/data/macro_expand_1.out.cpp index 6212601f713..e281d8d60f0 100644 --- a/tests/auto/cplusplus/preprocessor/data/macro_expand_1.out.cpp +++ b/tests/auto/cplusplus/preprocessor/data/macro_expand_1.out.cpp @@ -1,7 +1,7 @@ # 1 "data/macro_expand_1.cpp" -#gen true -# 1 "data/macro_expand_1.cpp" -class -#gen false + +# expansion begin 33,13 ~1 2:14 +class QString +# expansion end # 2 "data/macro_expand_1.cpp" - QString; + ; diff --git a/tests/auto/cplusplus/preprocessor/data/noPP.1.cpp b/tests/auto/cplusplus/preprocessor/data/noPP.1.cpp index 9df1cd12bc5..32b2797cee5 100644 --- a/tests/auto/cplusplus/preprocessor/data/noPP.1.cpp +++ b/tests/auto/cplusplus/preprocessor/data/noPP.1.cpp @@ -63,4 +63,3 @@ Write in C, write in C, Write in C, yeah, write in C. The government loves ADA, Write in C. - diff --git a/tests/auto/cplusplus/preprocessor/data/noPP.2.cpp b/tests/auto/cplusplus/preprocessor/data/noPP.2.cpp new file mode 100644 index 00000000000..60f2cd9596a --- /dev/null +++ b/tests/auto/cplusplus/preprocessor/data/noPP.2.cpp @@ -0,0 +1,26 @@ + +void hey(int a) { + int b = a + 10; + b; +} + +class hello +{ +public: + bool doit() { return true; } + bool dothat(); + void run(); +}; + +bool hello::dothat() +{ + bool should = true; + if (should) { + int i = 10; + while (i > 0) { + run(); + --i; + } + } + return false; +} diff --git a/tests/auto/cplusplus/preprocessor/data/poundpound.1.out.cpp b/tests/auto/cplusplus/preprocessor/data/poundpound.1.out.cpp index 40d92e888ac..96b07062101 100644 --- a/tests/auto/cplusplus/preprocessor/data/poundpound.1.out.cpp +++ b/tests/auto/cplusplus/preprocessor/data/poundpound.1.out.cpp @@ -1,8 +1,10 @@ # 1 "data/poundpound.1.cpp" struct QQ {}; -#gen true -# 3 "data/poundpound.1.cpp" + + + +# expansion begin 50,2 ~4 typedef QQ PPCC; -#gen false +# expansion end # 7 "data/poundpound.1.cpp" typedef PPCC RR; diff --git a/tests/auto/cplusplus/preprocessor/data/recursive.1.out.cpp b/tests/auto/cplusplus/preprocessor/data/recursive.1.out.cpp index 2beaba7c5b5..3f702f011ef 100644 --- a/tests/auto/cplusplus/preprocessor/data/recursive.1.out.cpp +++ b/tests/auto/cplusplus/preprocessor/data/recursive.1.out.cpp @@ -1,7 +1,11 @@ # 1 "data/recursive.1.cpp" -#gen true -# 1 "data/recursive.1.cpp" + + + +# expansion begin 25,1 ~1 b - a -#gen false +# expansion end +# expansion begin 27,1 ~1 +a +# expansion end # 6 "data/recursive.1.cpp" diff --git a/tests/auto/cplusplus/preprocessor/data/reserved.1.out.cpp b/tests/auto/cplusplus/preprocessor/data/reserved.1.out.cpp index e4339882b47..8eb99b63fbb 100644 --- a/tests/auto/cplusplus/preprocessor/data/reserved.1.out.cpp +++ b/tests/auto/cplusplus/preprocessor/data/reserved.1.out.cpp @@ -1,5 +1,8 @@ # 1 "data/reserved.1.cpp" -# 5 "data/reserved.1.cpp" + + + + int f() { foreach (QString &s, QStringList()) { doSomething(); diff --git a/tests/auto/cplusplus/preprocessor/preprocessor.pro b/tests/auto/cplusplus/preprocessor/preprocessor.pro index 9caff07d068..05c463833aa 100644 --- a/tests/auto/cplusplus/preprocessor/preprocessor.pro +++ b/tests/auto/cplusplus/preprocessor/preprocessor.pro @@ -3,11 +3,19 @@ include(../shared/shared.pri) SOURCES += tst_preprocessor.cpp OTHER_FILES = \ - data/noPP.1.cpp data/noPP.1.errors.txt \ - data/identifier-expansion.1.cpp data/identifier-expansion.1.out.cpp data/identifier-expansion.1.errors.txt \ - data/identifier-expansion.2.cpp data/identifier-expansion.2.out.cpp data/identifier-expansion.2.errors.txt \ - data/identifier-expansion.3.cpp data/identifier-expansion.3.out.cpp data/identifier-expansion.3.errors.txt \ - data/identifier-expansion.4.cpp data/identifier-expansion.4.out.cpp data/identifier-expansion.4.errors.txt \ - data/reserved.1.cpp data/reserved.1.out.cpp data/reserved.1.errors.txt \ - data/macro_expand.c data/macro_expand.out.c data/macro_expand.errors.txt \ - data/empty-macro.cpp data/empty-macro.out.cpp + data/noPP.1.cpp \ + data/noPP.2.cpp \ + data/identifier-expansion.1.cpp data/identifier-expansion.1.out.cpp \ + data/identifier-expansion.2.cpp data/identifier-expansion.2.out.cpp \ + data/identifier-expansion.3.cpp data/identifier-expansion.3.out.cpp \ + data/identifier-expansion.4.cpp data/identifier-expansion.4.out.cpp \ + data/identifier-expansion.5.cpp data/identifier-expansion.5.out.cpp \ + data/reserved.1.cpp data/reserved.1.out.cpp \ + data/recursive.1.cpp data/recursive.1.out.cpp \ + data/macro_expand.c data/macro_expand.out.c \ + data/macro_expand_1.cpp data/macro_expand_1.out.cpp \ + data/macro-test.cpp data/macro-test.out.cpp \ + data/poundpound.1.cpp data/poundpound.1.out.cpp \ + data/empty-macro.cpp data/empty-macro.out.cpp \ + data/empty-macro.2.cpp data/empty-macro.2.out.cpp \ + data/macro_pounder_fn.c diff --git a/tests/auto/cplusplus/preprocessor/tst_preprocessor.cpp b/tests/auto/cplusplus/preprocessor/tst_preprocessor.cpp index bc33d434579..f528c52a810 100644 --- a/tests/auto/cplusplus/preprocessor/tst_preprocessor.cpp +++ b/tests/auto/cplusplus/preprocessor/tst_preprocessor.cpp @@ -114,18 +114,26 @@ public: m_definedMacrosLine.append(macro.line()); } - virtual void passedMacroDefinitionCheck(unsigned /*offset*/, const Macro &/*macro*/) {} + virtual void passedMacroDefinitionCheck(unsigned /*offset*/, + unsigned /*line*/, + const Macro &/*macro*/) {} virtual void failedMacroDefinitionCheck(unsigned /*offset*/, const ByteArrayRef &/*name*/) {} + virtual void notifyMacroReference(unsigned offset, unsigned line, const Macro ¯o) + { + m_macroUsesLine[macro.name()].append(line); + m_expandedMacrosOffset.append(offset); + } + virtual void startExpandingMacro(unsigned offset, + unsigned line, const Macro ¯o, - const ByteArrayRef &originalText, const QVector &actuals = QVector()) { - m_expandedMacros.append(QByteArray(originalText.start(), originalText.length())); + m_expandedMacros.append(macro.name()); m_expandedMacrosOffset.append(offset); - m_macroUsesLine[macro.name()].append(m_env->currentLine); + m_macroUsesLine[macro.name()].append(line); m_macroArgsCount.append(actuals.size()); } @@ -137,8 +145,7 @@ public: virtual void stopSkippingBlocks(unsigned offset) { m_skippedBlocks.last().end = offset; } - virtual void sourceNeeded(QString &includedFileName, IncludeType mode, - unsigned line) + virtual void sourceNeeded(unsigned line, QString &includedFileName, IncludeType mode) { #if 1 m_recordedIncludes.append(Include(includedFileName, mode, line)); @@ -300,15 +307,14 @@ protected: } static QString simplified(QByteArray buf); -private /* not corrected yet */: - void macro_definition_lineno(); +private: + void compare_input_output(); private slots: - void defined(); - void defined_data(); - void va_args(); void named_va_args(); + void defined(); + void defined_data(); void empty_macro_args(); void macro_args_count(); void invalid_param_count(); @@ -318,14 +324,18 @@ private slots: void macro_arguments_notificatin(); void unfinished_function_like_macro_call(); void nasty_macro_expansion(); - void tstst(); - void test_file_builtin(); - + void glib_attribute(); + void builtin__FILE__(); void blockSkipping(); void includes_1(); - + void dont_eagerly_expand(); + void dont_eagerly_expand_data(); void comparisons_data(); void comparisons(); + void comments_within(); + void comments_within_data(); + void multitokens_argument(); + void multitokens_argument_data(); }; // Remove all #... lines, and 'simplify' string, to allow easily comparing the result @@ -534,38 +544,72 @@ void tst_Preprocessor::macro_uses_lines() << buffer.lastIndexOf("ENABLE(LESS)")); } -void tst_Preprocessor::macro_definition_lineno() +void tst_Preprocessor::multitokens_argument_data() { - Client *client = 0; // no client. - Environment env; - Preprocessor preprocess(client, &env); - QByteArray preprocessed = preprocess.run(QLatin1String(""), - QByteArray("#define foo(ARGS) int f(ARGS)\n" - "foo(int a);\n")); - QVERIFY(preprocessed.contains("#gen true\n# 2 \"\"\nint f")); + QTest::addColumn("input"); + QTest::addColumn("output"); - preprocessed = preprocess.run(QLatin1String(""), - QByteArray("#define foo(ARGS) int f(ARGS)\n" - "foo(int a)\n" - ";\n")); - QVERIFY(preprocessed.contains("#gen true\n# 2 \"\"\nint f")); + QByteArray original; + QByteArray expected; - preprocessed = preprocess.run(QLatin1String(""), - QByteArray("#define foo(ARGS) int f(ARGS)\n" - "foo(int \n" - " a);\n")); - QVERIFY(preprocessed.contains("#gen true\n# 2 \"\"\nint f")); + original = + "#define foo(ARGS) int f(ARGS)\n" + "foo(int a);\n"; + expected = + "# 1 \"\"\n" + "\n" + "# expansion begin 30,3 ~3 2:4 2:8 ~1\n" + "int f(int a)\n" + "# expansion end\n" + "# 2 \"\"\n" + " ;\n"; + QTest::newRow("case 1") << original << expected; - preprocessed = preprocess.run(QLatin1String(""), - QByteArray("#define foo int f\n" - "foo;\n")); - QVERIFY(preprocessed.contains("#gen true\n# 2 \"\"\nint f")); + original = + "#define foo(ARGS) int f(ARGS)\n" + "foo(int \n" + " a);\n"; + expected = + "# 1 \"\"\n" + "\n" + "# expansion begin 30,3 ~3 2:4 3:4 ~1\n" + "int f(int a)\n" + "# expansion end\n" + "# 3 \"\"\n" + " ;\n"; + QTest::newRow("case 2") << original << expected; - preprocessed = preprocess.run(QLatin1String(""), - QByteArray("#define foo int f\n" - "foo\n" - ";\n")); - QVERIFY(preprocessed.contains("#gen true\n# 2 \"\"\nint f")); + original = + "#define foo(ARGS) int f(ARGS)\n" + "foo(int a = 0);\n"; + expected = + "# 1 \"\"\n" + "\n" + "# expansion begin 30,3 ~3 2:4 2:8 2:10 2:12 ~1\n" + "int f(int a = 0)\n" + "# expansion end\n" + "# 2 \"\"\n" + " ;\n"; + QTest::newRow("case 3") << original << expected; + + original = + "#define foo(X) int f(X = 0)\n" + "foo(int \n" + " a);\n"; + expected = + "# 1 \"\"\n" + "\n" + "# expansion begin 28,3 ~3 2:4 3:4 ~3\n" + "int f(int a = 0)\n" + "# expansion end\n" + "# 3 \"\"\n" + " ;\n"; + QTest::newRow("case 4") << original << expected; +} + +void tst_Preprocessor::multitokens_argument() +{ + compare_input_output(); } void tst_Preprocessor::objmacro_expanding_as_fnmacro_notification() @@ -608,9 +652,17 @@ void tst_Preprocessor::unfinished_function_like_macro_call() Preprocessor preprocess(client, &env); QByteArray preprocessed = preprocess.run(QLatin1String(""), - QByteArray("\n#define foo(a,b) a + b" - "\nfoo(1, 2\n")); - QByteArray expected__("# 1 \"\"\n\n\n 1\n#gen true\n# 2 \"\"\n+\n#gen false\n# 3 \"\"\n 2\n"); + QByteArray("\n" + "#define foo(a,b) a + b\n" + "foo(1, 2\n")); + QByteArray expected__("# 1 \"\"\n" + "\n" + "\n" + "# expansion begin 24,3 3:4 ~1 3:7\n" + "1 + 2\n" + "# expansion end\n" + "# 4 \"\"\n"); + // DUMP_OUTPUT(preprocessed); QCOMPARE(preprocessed, expected__); } @@ -668,12 +720,10 @@ void tst_Preprocessor::nasty_macro_expansion() QVERIFY(!preprocessed.contains("FIELD32")); } -void tst_Preprocessor::tstst() +void tst_Preprocessor::glib_attribute() { - Client *client = 0; // no client. Environment env; - - Preprocessor preprocess(client, &env); + Preprocessor preprocess(0, &env); QByteArray preprocessed = preprocess.run( QLatin1String(""), QByteArray("\n" @@ -681,15 +731,14 @@ void tst_Preprocessor::tstst() "namespace std _GLIBCXX_VISIBILITY(default) {\n" "}\n" )); - const QByteArray result____ ="# 1 \"\"\n\n\n" + const QByteArray result____ = + "# 1 \"\"\n" + "\n" + "\n" "namespace std\n" - "#gen true\n" - "# 2 \"\"\n" - "__attribute__ ((__visibility__ (\n" - "\"default\"\n" - "# 2 \"\"\n" - ")))\n" - "#gen false\n" + "# expansion begin 85,19 ~9\n" + "__attribute__ ((__visibility__ (\"default\")))\n" + "# expansion end\n" "# 3 \"\"\n" " {\n" "}\n"; @@ -698,7 +747,7 @@ void tst_Preprocessor::tstst() QCOMPARE(preprocessed, result____); } -void tst_Preprocessor::test_file_builtin() +void tst_Preprocessor::builtin__FILE__() { Client *client = 0; // no client. Environment env; @@ -710,13 +759,8 @@ void tst_Preprocessor::test_file_builtin() )); const QByteArray result____ = "# 1 \"some-file.c\"\n" - "const char *f =\n" - "#gen true\n" - "# 1 \"some-file.c\"\n" - "\"some-file.c\"\n" - "#gen false\n" - "# 2 \"some-file.c\"\n" - ; + "const char *f = \"some-file.c\"\n"; + QCOMPARE(preprocessed, result____); } @@ -727,6 +771,7 @@ void tst_Preprocessor::comparisons_data() QTest::addColumn("errorfile"); QTest::newRow("do nothing") << "noPP.1.cpp" << "noPP.1.cpp" << ""; + QTest::newRow("no PP 2") << "noPP.2.cpp" << "noPP.2.cpp" << ""; QTest::newRow("identifier-expansion 1") << "identifier-expansion.1.cpp" << "identifier-expansion.1.out.cpp" << ""; QTest::newRow("identifier-expansion 2") @@ -766,6 +811,7 @@ void tst_Preprocessor::comparisons() QByteArray errors; QByteArray preprocessed = preprocess(infile, &errors, infile == outfile); + // DUMP_OUTPUT(preprocessed); if (!outfile.isEmpty()) { @@ -976,6 +1022,176 @@ void tst_Preprocessor::defined_data() "#endif\n"; } +void tst_Preprocessor::dont_eagerly_expand_data() +{ + QTest::addColumn("input"); + QTest::addColumn("output"); + + QByteArray original; + QByteArray expected; + + // Expansion must be processed upon invocation of the macro. Therefore a particular + // identifier within a define must not be expanded (in the case it matches an + // already known macro) during the processor directive handling, but only when + // it's actually "used". Naturally, if it's still not replaced after an invocation + // it should then be expanded. This is consistent with clang and gcc for example. + + original = "#define T int\n" + "#define FOO(T) T\n" + "FOO(double)\n"; + expected = + "# 1 \"\"\n" + "\n" + "\n" + "# expansion begin 31,3 3:4\n" + "double\n" + "# expansion end\n" + "# 4 \"\"\n"; + QTest::newRow("case 1") << original << expected; + + original = "#define T int\n" + "#define FOO(X) T\n" + "FOO(double)\n"; + expected = + "# 1 \"\"\n" + "\n" + "\n" + "# expansion begin 31,3 ~1\n" + "int\n" + "# expansion end\n" + "# 4 \"\"\n"; + QTest::newRow("case 2") << original << expected; +} + +void tst_Preprocessor::dont_eagerly_expand() +{ + compare_input_output(); +} + +void tst_Preprocessor::comments_within() +{ + compare_input_output(); +} + +void tst_Preprocessor::comments_within_data() +{ + QTest::addColumn("input"); + QTest::addColumn("output"); + + QByteArray original; + QByteArray expected; + + original = "#define FOO int x;\n" + "\n" + " // comment\n" + " // comment\n" + " // comment\n" + " // comment\n" + "FOO\n" + "x = 10\n"; + expected = + "# 1 \"\"\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "# expansion begin 76,3 ~3\n" + "int x;\n" + "# expansion end\n" + "# 8 \"\"\n" + "x = 10\n"; + QTest::newRow("case 1") << original << expected; + + + original = "#define FOO int x;\n" + "\n" + " /* comment\n" + " comment\n" + " comment\n" + " comment */\n" + "FOO\n" + "x = 10\n"; + expected = + "# 1 \"\"\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "# expansion begin 79,3 ~3\n" + "int x;\n" + "# expansion end\n" + "# 8 \"\"\n" + "x = 10\n"; + QTest::newRow("case 2") << original << expected; + + + original = "#define FOO int x;\n" + "\n" + " // comment\n" + " // comment\n" + " // comment\n" + " // comment\n" + "FOO\n" + "// test\n" + "// test again\n" + "x = 10\n"; + expected = + "# 1 \"\"\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "# expansion begin 76,3 ~3\n" + "int x;\n" + "# expansion end\n" + "# 10 \"\"\n" + "x = 10\n"; + QTest::newRow("case 3") << original << expected; + + + original = "#define FOO int x;\n" + "\n" + " /* comment\n" + " comment\n" + " comment\n" + " comment */\n" + "FOO\n" + "/* \n" + "*/\n" + "x = 10\n"; + expected = + "# 1 \"\"\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "# expansion begin 79,3 ~3\n" + "int x;\n" + "# expansion end\n" + "# 10 \"\"\n" + "x = 10\n"; + QTest::newRow("case 4") << original << expected; +} + +void tst_Preprocessor::compare_input_output() +{ + QFETCH(QByteArray, input); + QFETCH(QByteArray, output); + + Environment env; + Preprocessor preprocess(0, &env); + QByteArray prep = preprocess.run(QLatin1String(""), input); + QCOMPARE(output, prep); +} + QTEST_APPLESS_MAIN(tst_Preprocessor) #include "tst_preprocessor.moc"