feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

C++: Small fixes to the token buffer.

Browse Source 
- Fixed blocking macro check
- Enabled buffer "compression": when tokens are generated and no new
macro is being blocked, then prepend the tokens to the previous buffer.
This happens a lot when undo-ing look-ahead.
- Added documentation

Change-Id: I6fa816d94ce4696e473bdbc4f3bf477d77e4dd51
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@digia.com>
This commit is contained in:
Erik Verbruggen
2012-12-21 10:35:38 +01:00
parent eb09c87d99
commit 4e0880bde4
1 changed files with 25 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/libs/cplusplus/pp-engine.cpp
View File

@@ -106,6 +106,14 @@ typedef ScopedSwap<unsigned> ScopedUnsignedSwap;
namespace CPlusPlus {
namespace Internal {
/// Buffers tokens for the Preprocessor::lex() to read next. Do not use this
/// class directly, but use Preprocessor::State::pushTokenBuffer .
///
/// New tokens are added when undoing look-ahead, or after expanding a macro.
/// When macro expansion happened, the macro is passed in, and blocked until
/// all tokens generated by it (and by subsequent expansion of those generated
/// tokens) are read from the buffer. See Preprocessor::lex() for details on
/// exactly when the buffer (and subsequently a blocking macro) is removed.
struct TokenBuffer
{
std::deque<PPToken> tokens;
@@ -121,8 +129,9 @@ struct TokenBuffer
return false;
for (const TokenBuffer *it = this; it; it = it->next)
if (it->macro == macro && it->macro->name() == macro->name())
return true;
if (it->macro)
if (it->macro == macro || (it->macro->name() == macro->name()))
return true;
return false;
}
};
@@ -555,22 +564,21 @@ Preprocessor::State::State()
m_expansionResult.reserve(256);
}
//#define COMPRESS_TOKEN_BUFFER
#define COMPRESS_TOKEN_BUFFER
void Preprocessor::State::pushTokenBuffer(const PPToken *start, const PPToken *end, const Macro *macro)
{
if (m_tokenBufferDepth <= MAX_TOKEN_BUFFER_DEPTH) {
#ifdef COMPRESS_TOKEN_BUFFER
// This does not work correctly for boost's preprocessor library, or that library exposes a bug in the code.
if (macro || !m_tokenBuffer) {
// If there is a new blocking macro (or no token buffer yet), create
// one.
m_tokenBuffer = new TokenBuffer(start, end, macro, m_tokenBuffer);
++m_tokenBufferDepth;
} else {
// No new blocking macro is passed in, so tokens can be prepended to
// the existing buffer.
m_tokenBuffer->tokens.insert(m_tokenBuffer->tokens.begin(), start, end);
}
unsigned tkCount = 0;
for (TokenBuffer *it = m_tokenBuffer; it; it = m_tokenBuffer->next)
tkCount += it->tokens.size();
qDebug()<<"New depth:" << m_tokenBufferDepth << "with total token count:" << tkCount;
#else
m_tokenBuffer = new TokenBuffer(start, end, macro, m_tokenBuffer);
++m_tokenBufferDepth;
@@ -693,13 +701,22 @@ void Preprocessor::lex(PPToken *tk)
{
_Lagain:
if (m_state.m_tokenBuffer) {
// There is a token buffer, so read from there.
if (m_state.m_tokenBuffer->tokens.empty()) {
// The token buffer is empty, so pop it, and start over.
m_state.popTokenBuffer();
goto _Lagain;
}
*tk = m_state.m_tokenBuffer->tokens.front();
m_state.m_tokenBuffer->tokens.pop_front();
// The token buffer might now be empty. We leave it in, because the
// token we just read might expand into new tokens, or might be a call
// to the macro that generated this token. In either case, the macro
// that generated the token still needs to be blocked (!), which is
// recorded in the token buffer. Removing the blocked macro and the
// empty token buffer happens the next time that this method is called.
} else {
// No token buffer, so have the lexer scan the next token.
tk->setSource(m_state.m_source);
m_state.m_lexer->scan(tk);
}