diff --git a/src/libs/qmljs/qmljsscanner.cpp b/src/libs/qmljs/qmljsscanner.cpp index 32fa532ebc9..3328a71c041 100644 --- a/src/libs/qmljs/qmljsscanner.cpp +++ b/src/libs/qmljs/qmljsscanner.cpp @@ -183,6 +183,22 @@ static inline void setRegexpMayFollow(int *state, bool on) *state = (on ? Scanner::RegexpMayFollow : 0) | (*state & ~Scanner::RegexpMayFollow); } +static inline int templateExpressionDepth(int state) +{ + if ((state & Scanner::TemplateExpressionOpenBracesMask) == 0) + return 0; + if ((state & (Scanner::TemplateExpressionOpenBracesMask3 | Scanner::TemplateExpressionOpenBracesMask4)) == 0) { + if ((state & Scanner::TemplateExpressionOpenBracesMask2) == 0) + return 1; + else + return 2; + } + if ((state & Scanner::TemplateExpressionOpenBracesMask4) == 0) + return 3; + else + return 4; +} + QList Scanner::operator()(const QString &text, int startState) { _state = startState; @@ -190,6 +206,45 @@ QList Scanner::operator()(const QString &text, int startState) int index = 0; + auto scanTemplateString = [&index, &text, &tokens, this](int startShift = 0){ + const QChar quote = QLatin1Char('`'); + const int start = index + startShift; + while (index < text.length()) { + const QChar ch = text.at(index); + + if (ch == quote) + break; + else if (ch == QLatin1Char('$') && index + 1 < text.length() && text.at(index + 1) == QLatin1Char('{')) { + tokens.append(Token(start, index - start, Token::String)); + tokens.append(Token(index, 2, Token::Delimiter)); + index += 2; + setRegexpMayFollow(&_state, true); + setMultiLineState(&_state, Normal); + int depth = templateExpressionDepth(_state); + if (depth == 4) { + qWarning() << "QQmljs::Dom::Scanner reached maximum nesting of template expressions (4), parsing will fail"; + } else { + _state |= 1 << (4 + depth * 7); + } + return; + } else if (ch == QLatin1Char('\\') && index + 1 < text.length()) + index += 2; + else + ++index; + } + + if (index < text.length()) { + setMultiLineState(&_state, Normal); + ++index; + // good one + } else { + setMultiLineState(&_state, MultiLineStringBQuote); + } + + tokens.append(Token(start, index - start, Token::String)); + setRegexpMayFollow(&_state, false); + }; + if (multiLineState(_state) == MultiLineComment) { int start = -1; while (index < text.length()) { @@ -233,8 +288,14 @@ QList Scanner::operator()(const QString &text, int startState) if (start < index) tokens.append(Token(start, index - start, Token::String)); setRegexpMayFollow(&_state, false); + } else if (multiLineState(_state) == MultiLineStringBQuote) { + scanTemplateString(); } + auto braceCounterOffset = [](int templateDepth) { + return FlagsBits + (templateDepth - 1) * BraceCounterBits; + }; + while (index < text.length()) { const QChar ch = text.at(index); @@ -273,6 +334,9 @@ QList Scanner::operator()(const QString &text, int startState) tokens.append(Token(index, end - index, Token::RegExp)); index = end; setRegexpMayFollow(&_state, false); + } else if (la == QLatin1Char('=')) { + tokens.append(Token(index, 2, Token::Delimiter)); + index += 2; } else { tokens.append(Token(index++, 1, Token::Delimiter)); setRegexpMayFollow(&_state, true); @@ -280,7 +344,6 @@ QList Scanner::operator()(const QString &text, int startState) break; case '\'': - case '`': case '"': { const QChar quote = ch; const int start = index; @@ -310,6 +373,11 @@ QList Scanner::operator()(const QString &text, int startState) setRegexpMayFollow(&_state, false); } break; + case '`': { + ++index; + scanTemplateString(-1); + } break; + case '.': if (la.isDigit()) { const int start = index; @@ -343,15 +411,38 @@ QList Scanner::operator()(const QString &text, int startState) setRegexpMayFollow(&_state, false); break; - case '{': + case '{':{ tokens.append(Token(index++, 1, Token::LeftBrace)); setRegexpMayFollow(&_state, true); - break; + int depth = templateExpressionDepth(_state); + if (depth > 0) { + int shift = braceCounterOffset(depth); + int mask = Scanner::TemplateExpressionOpenBracesMask0 << shift; + if ((_state & mask) == mask) { + qWarning() << "QQmljs::Dom::Scanner reached maximum open braces of template expressions (127), parsing will fail"; + } else { + _state = (_state & ~mask) | (((Scanner::TemplateExpressionOpenBracesMask0 & (_state >> shift)) + 1) << shift); + } + } + } break; - case '}': - tokens.append(Token(index++, 1, Token::RightBrace)); + case '}': { setRegexpMayFollow(&_state, false); - break; + int depth = templateExpressionDepth(_state); + if (depth > 0) { + int shift = braceCounterOffset(depth); + int s = _state; + int nBraces = Scanner::TemplateExpressionOpenBracesMask0 & (s >> shift); + int mask = Scanner::TemplateExpressionOpenBracesMask0 << shift; + _state = (s & ~mask) | ((nBraces - 1) << shift); + if (nBraces == 1) { + tokens.append(Token(index++, 1, Token::Delimiter)); + scanTemplateString(); + break; + } + } + tokens.append(Token(index++, 1, Token::RightBrace)); + } break; case ';': tokens.append(Token(index++, 1, Token::Semicolon)); @@ -370,6 +461,32 @@ QList Scanner::operator()(const QString &text, int startState) case '+': case '-': + case '<': + if (la == ch || la == QLatin1Char('=')) { + tokens.append(Token(index, 2, Token::Delimiter)); + index += 2; + } else { + tokens.append(Token(index++, 1, Token::Delimiter)); + } + setRegexpMayFollow(&_state, true); + break; + + case '>': + if (la == ch && index + 2 < text.length() && text.at(index + 2) == ch) { + tokens.append(Token(index, 2, Token::Delimiter)); + index += 3; + } else if (la == ch || la == QLatin1Char('=')) { + tokens.append(Token(index, 2, Token::Delimiter)); + index += 2; + } else { + tokens.append(Token(index++, 1, Token::Delimiter)); + } + setRegexpMayFollow(&_state, true); + break; + + case '|': + case '=': + case '&': if (la == ch) { tokens.append(Token(index, 2, Token::Delimiter)); index += 2; diff --git a/src/libs/qmljs/qmljsscanner.h b/src/libs/qmljs/qmljsscanner.h index 0cd8c69d7db..523616de3ef 100644 --- a/src/libs/qmljs/qmljsscanner.h +++ b/src/libs/qmljs/qmljsscanner.h @@ -62,23 +62,60 @@ public: inline bool is(int k) const { return k == kind; } inline bool isNot(int k) const { return k != kind; } + static int compare(const Token &t1, const Token &t2) { + if (int c = t1.offset - t2.offset) + return c; + if (int c = t1.length - t2.length) + return c; + return int(t1.kind) - int(t2.kind); + } + public: - int offset; - int length; - Kind kind; + int offset = 0; + int length = 0; + Kind kind = EndOfFile; }; +inline int operator == (const Token &t1, const Token &t2) { + return Token::compare(t1, t2) == 0; +} +inline int operator != (const Token &t1, const Token &t2) { + return Token::compare(t1, t2) != 0; +} + class QMLJS_EXPORT Scanner { public: + enum { + FlagsBits = 4, + BraceCounterBits = 7 + }; enum { Normal = 0, MultiLineComment = 1, MultiLineStringDQuote = 2, MultiLineStringSQuote = 3, - MultiLineMask = 3, + MultiLineStringBQuote = 4, + MultiLineMask = 7, - RegexpMayFollow = 4 // flag that may be combined with the above + RegexpMayFollow = 8, // flag that may be combined with the above + + // templates can be nested, which means that the scanner/lexer cannot + // be a simple state machine anymore, but should have a stack to store + // the state (the number of open braces in the current template + // string). + // The lexer stare is currently stored in an int, so we abuse that and + // store a the number of open braces (maximum 0x7f = 127) for at most 5 + // nested templates in the int after the flags for the multiline + // comments and strings. + TemplateExpression = 0x1 << 4, + TemplateExpressionOpenBracesMask0 = 0x7F, + TemplateExpressionOpenBracesMask1 = 0x7F << 4, + TemplateExpressionOpenBracesMask2 = 0x7F << 11, + TemplateExpressionOpenBracesMask3 = 0x7F << 18, + TemplateExpressionOpenBracesMask4 = 0x7F << 25, + TemplateExpressionOpenBracesMask = TemplateExpressionOpenBracesMask1 | TemplateExpressionOpenBracesMask2 + | TemplateExpressionOpenBracesMask3 | TemplateExpressionOpenBracesMask4 }; Scanner(); diff --git a/tests/auto/qml/reformatter/jssyntax.js b/tests/auto/qml/reformatter/jssyntax.js index b6c2a9e9054..2d46f597c80 100644 --- a/tests/auto/qml/reformatter/jssyntax.js +++ b/tests/auto/qml/reformatter/jssyntax.js @@ -5,6 +5,11 @@ var a_var = 1 let a_let = 2 const a_const = 3 +const tmpl = `template` + `t${i + 6}` + `t${i + `nested${i}`}` + `t${function () { + return 5 +}()}` + `t\${i} +${i + 2}` + function foo(a, b) { x = 15 x += 4