From 25f37732b01e292cd623b47f431a28975e3c9cea Mon Sep 17 00:00:00 2001 From: Christian Kamm Date: Mon, 19 Sep 2011 14:09:26 +0200 Subject: [PATCH] QmlJS: Update parser from Qt5. Change-Id: I26a5fb6a1eb99a777e4f1e01fb7b19e559c9bce7 Reviewed-on: http://codereview.qt-project.org/5143 Reviewed-by: Roberto Raggi --- src/libs/qmljs/parser/gen-parser.sh | 4 - src/libs/qmljs/parser/qmldirparser.cpp | 7 +- src/libs/qmljs/parser/qmljs.g | 12 +- src/libs/qmljs/parser/qmljslexer.cpp | 184 +++++++++++++++++++------ src/libs/qmljs/parser/qmljslexer_p.h | 33 +++++ src/libs/qmljs/parser/qmljsparser.cpp | 12 +- src/libs/qmljs/parser/qmlutils_p.h | 82 ----------- 7 files changed, 182 insertions(+), 152 deletions(-) delete mode 100644 src/libs/qmljs/parser/qmlutils_p.h diff --git a/src/libs/qmljs/parser/gen-parser.sh b/src/libs/qmljs/parser/gen-parser.sh index 1995ad9cea3..b36bdaa3330 100755 --- a/src/libs/qmljs/parser/gen-parser.sh +++ b/src/libs/qmljs/parser/gen-parser.sh @@ -10,10 +10,6 @@ for i in $QTDIR/src/declarative/qml/qdeclarative{error.{h,cpp},dirparser{_p.h,.c sed -f $me/cmd.sed $i > $me/$(echo $(basename $i) | sed s/qdeclarative/qml/) done -for i in $QTDIR/src/declarative/qml/ftw/qdeclarativeutils_p.h; do - sed -f $me/cmd.sed $i > $me/$(echo $(basename $i) | sed s/qdeclarative/qml/) -done - # export QmlDirParser perl -p -0777 -i -e 's/QT_BEGIN_NAMESPACE\n\nclass QmlError;\nclass QmlDirParser/#include "qmljsglobal_p.h"\n\nQT_BEGIN_NAMESPACE\n\nclass QmlError;\nclass QML_PARSER_EXPORT QmlDirParser/' qmldirparser_p.h # replace qmlglobal_p.h include with needed declaration diff --git a/src/libs/qmljs/parser/qmldirparser.cpp b/src/libs/qmljs/parser/qmldirparser.cpp index 3914ebc1b37..593fbb81c50 100644 --- a/src/libs/qmljs/parser/qmldirparser.cpp +++ b/src/libs/qmljs/parser/qmldirparser.cpp @@ -33,7 +33,6 @@ #include "qmldirparser_p.h" #include "qmlerror.h" bool Qml_isFileCaseCorrect(const QString &) { return true; } -#include #include #include @@ -132,9 +131,9 @@ bool QmlDirParser::parse() while (index != length) { const QChar ch = line.at(index); - if (QmlUtils::isSpace(ch)) { + if (ch.isSpace()) { do { ++index; } - while (index != length && QmlUtils::isSpace(line.at(index))); + while (index != length && line.at(index).isSpace()); } else if (ch == QLatin1Char('#')) { // recognized a comment @@ -144,7 +143,7 @@ bool QmlDirParser::parse() const int start = index; do { ++index; } - while (index != length && !QmlUtils::isSpace(line.at(index))); + while (index != length && !line.at(index).isSpace()); const QString lexeme = line.mid(start, index - start); diff --git a/src/libs/qmljs/parser/qmljs.g b/src/libs/qmljs/parser/qmljs.g index 1542c2d3013..c221b0c64fd 100644 --- a/src/libs/qmljs/parser/qmljs.g +++ b/src/libs/qmljs/parser/qmljs.g @@ -125,7 +125,7 @@ **************************************************************************/ #include -#include +#include #include @@ -398,14 +398,6 @@ void Parser::reallocateStack() string_stack = reinterpret_cast (qRealloc(string_stack, stack_size * sizeof(QStringRef))); } -inline static bool automatic(Engine *driver, int token) -{ - return token == $table::T_RBRACE - || token == 0 - || driver->lexer()->prevTerminator(); -} - - Parser::Parser(Engine *engine): driver(engine), pool(engine->pool()), @@ -2881,7 +2873,7 @@ PropertyNameAndValueListOpt: PropertyNameAndValueList ; const int errorState = state_stack[tos]; // automatic insertion of `;' - if (yytoken != -1 && t_action(errorState, T_AUTOMATIC_SEMICOLON) && automatic(driver, yytoken)) { + if (yytoken != -1 && t_action(errorState, T_AUTOMATIC_SEMICOLON) && lexer->canInsertAutomaticSemicolon(yytoken)) { SavedToken &tk = token_buffer[0]; tk.token = yytoken; tk.dval = yylval; diff --git a/src/libs/qmljs/parser/qmljslexer.cpp b/src/libs/qmljs/parser/qmljslexer.cpp index 8bbc0f0a7f7..a35f552b6a1 100644 --- a/src/libs/qmljs/parser/qmljslexer.cpp +++ b/src/libs/qmljs/parser/qmljslexer.cpp @@ -38,37 +38,18 @@ #include #include -#ifndef QMLJS_NO_FTW -#include -#else -namespace QmlUtils { -inline bool isUpper(const QChar &qc){ return qc.isUpper(); } -inline bool isLower(const QChar &qc) { return qc.isLower(); } -inline bool isLetter(const QChar &qc) { return qc.isLetter(); } -inline bool isDigit(const QChar &qc) { return qc.isDigit(); } -inline bool isLetterOrNumber(const QChar &qc) { return qc.isLetterOrNumber(); } -inline bool isSpace(const QChar &qc) { return qc.isSpace(); } -} // namespace QmlUtils -#endif - QT_BEGIN_NAMESPACE Q_CORE_EXPORT double qstrtod(const char *s00, char const **se, bool *ok); QT_END_NAMESPACE using namespace QmlJS; -enum RegExpFlag { - Global = 0x01, - IgnoreCase = 0x02, - Multiline = 0x04 -}; - -static int flagFromChar(const QChar &ch) +static int regExpFlagFromChar(const QChar &ch) { switch (ch.unicode()) { - case 'g': return Global; - case 'i': return IgnoreCase; - case 'm': return Multiline; + case 'g': return Lexer::RegExp_Global; + case 'i': return Lexer::RegExp_IgnoreCase; + case 'm': return Lexer::RegExp_Multiline; } return 0; } @@ -115,6 +96,7 @@ Lexer::Lexer(Engine *engine) , _prohibitAutomaticSemicolon(false) , _restrictedKeyword(false) , _terminator(false) + , _followsClosingBrace(false) , _delimited(false) , _qmlMode(true) { @@ -164,6 +146,7 @@ void Lexer::setCode(const QString &code, int lineno, bool qmlMode) _prohibitAutomaticSemicolon = false; _restrictedKeyword = false; _terminator = false; + _followsClosingBrace = false; _delimited = false; } @@ -179,12 +162,15 @@ void Lexer::scanChar() int Lexer::lex() { + const int previousTokenKind = _tokenKind; + _tokenSpell = QStringRef(); _tokenKind = scanToken(); _tokenLength = _codePtr - _tokenStartPtr - 1; _delimited = false; _restrictedKeyword = false; + _followsClosingBrace = (previousTokenKind == T_RBRACE); // update the flags switch (_tokenKind) { @@ -285,7 +271,7 @@ again: _validTokenText = false; _tokenLinePtr = _lastLinePtr; - while (QmlUtils::isSpace(_char)) { + while (_char.isSpace()) { if (_char == QLatin1Char('\n')) { _tokenLinePtr = _codePtr; @@ -424,19 +410,19 @@ again: return T_DIVIDE_; case '.': - if (QmlUtils::isDigit(_char)) { + if (_char.isDigit()) { QVarLengthArray chars; chars.append(ch.unicode()); // append the `.' - while (QmlUtils::isDigit(_char)) { + while (_char.isDigit()) { chars.append(_char.unicode()); scanChar(); } if (_char == QLatin1Char('e') || _char == QLatin1Char('E')) { - if (QmlUtils::isDigit(_codePtr[0]) || ((_codePtr[0] == QLatin1Char('+') || _codePtr[0] == QLatin1Char('-')) && - QmlUtils::isDigit(_codePtr[1]))) { + if (_codePtr[0].isDigit() || ((_codePtr[0] == QLatin1Char('+') || _codePtr[0] == QLatin1Char('-')) && + _codePtr[1].isDigit())) { chars.append(_char.unicode()); scanChar(); // consume `e' @@ -446,7 +432,7 @@ again: scanChar(); // consume the sign } - while (QmlUtils::isDigit(_char)) { + while (_char.isDigit()) { chars.append(_char.unicode()); scanChar(); } @@ -672,7 +658,7 @@ again: } default: - if (QmlUtils::isLetter(ch) || ch == QLatin1Char('$') || ch == QLatin1Char('_') || (ch == QLatin1Char('\\') && _char == QLatin1Char('u'))) { + if (ch.isLetter() || ch == QLatin1Char('$') || ch == QLatin1Char('_') || (ch == QLatin1Char('\\') && _char == QLatin1Char('u'))) { bool identifierWithEscapeChars = false; if (ch == QLatin1Char('\\')) { identifierWithEscapeChars = true; @@ -687,7 +673,7 @@ again: } } while (true) { - if (QmlUtils::isLetterOrNumber(_char) || _char == QLatin1Char('$') || _char == QLatin1Char('_')) { + if (_char.isLetterOrNumber() || _char == QLatin1Char('$') || _char == QLatin1Char('_')) { if (identifierWithEscapeChars) _tokenText += _char; @@ -726,13 +712,13 @@ again: return kind; } } - } else if (QmlUtils::isDigit(ch)) { + } else if (ch.isDigit()) { if (ch != QLatin1Char('0')) { double integer = ch.unicode() - '0'; QChar n = _char; const QChar *code = _codePtr; - while (QmlUtils::isDigit(n)) { + while (n.isDigit()) { integer = integer * 10 + (n.unicode() - '0'); n = *code++; } @@ -766,7 +752,7 @@ again: } // decimal integer literal - while (QmlUtils::isDigit(_char)) { + while (_char.isDigit()) { chars.append(_char.unicode()); scanChar(); // consume the digit } @@ -775,14 +761,14 @@ again: chars.append(_char.unicode()); scanChar(); // consume `.' - while (QmlUtils::isDigit(_char)) { + while (_char.isDigit()) { chars.append(_char.unicode()); scanChar(); } if (_char == QLatin1Char('e') || _char == QLatin1Char('E')) { - if (QmlUtils::isDigit(_codePtr[0]) || ((_codePtr[0] == QLatin1Char('+') || _codePtr[0] == QLatin1Char('-')) && - QmlUtils::isDigit(_codePtr[1]))) { + if (_codePtr[0].isDigit() || ((_codePtr[0] == QLatin1Char('+') || _codePtr[0] == QLatin1Char('-')) && + _codePtr[1].isDigit())) { chars.append(_char.unicode()); scanChar(); // consume `e' @@ -792,15 +778,15 @@ again: scanChar(); // consume the sign } - while (QmlUtils::isDigit(_char)) { + while (_char.isDigit()) { chars.append(_char.unicode()); scanChar(); } } } } else if (_char == QLatin1Char('e') || _char == QLatin1Char('E')) { - if (QmlUtils::isDigit(_codePtr[0]) || ((_codePtr[0] == QLatin1Char('+') || _codePtr[0] == QLatin1Char('-')) && - QmlUtils::isDigit(_codePtr[1]))) { + if (_codePtr[0].isDigit() || ((_codePtr[0] == QLatin1Char('+') || _codePtr[0] == QLatin1Char('-')) && + _codePtr[1].isDigit())) { chars.append(_char.unicode()); scanChar(); // consume `e' @@ -810,7 +796,7 @@ again: scanChar(); // consume the sign } - while (QmlUtils::isDigit(_char)) { + while (_char.isDigit()) { chars.append(_char.unicode()); scanChar(); } @@ -862,7 +848,7 @@ bool Lexer::scanRegExp(RegExpBodyPrefix prefix) // scan the flags _patternFlags = 0; while (isIdentLetter(_char)) { - int flag = flagFromChar(_char); + int flag = regExpFlagFromChar(_char); if (flag == 0) { _errorMessage = QCoreApplication::translate("QmlParser", "Invalid regular expression flag '%0'") .arg(QChar(_char)); @@ -969,6 +955,11 @@ bool Lexer::isOctalDigit(ushort c) return (c >= '0' && c <= '7'); } +int Lexer::tokenKind() const +{ + return _tokenKind; +} + int Lexer::tokenOffset() const { return _tokenStartPtr - _code.unicode(); @@ -1048,4 +1039,113 @@ bool Lexer::prevTerminator() const return _terminator; } +bool Lexer::followsClosingBrace() const +{ + return _followsClosingBrace; +} + +bool Lexer::canInsertAutomaticSemicolon(int token) const +{ + return token == T_RBRACE + || token == EOF_SYMBOL + || _terminator + || _followsClosingBrace; +} + +bool Lexer::scanDirectives(Directives *directives) +{ + if (_qmlMode) { + // the directives are a Javascript-only extension. + return false; + } + + lex(); // fetch the first token + + if (_tokenKind != T_DOT) + return true; + + do { + lex(); // skip T_DOT + + const int lineNumber = tokenStartLine(); + + if (! (_tokenKind == T_IDENTIFIER || _tokenKind == T_RESERVED_WORD)) + return false; // expected a valid QML/JS directive + + const QString directiveName = tokenText(); + + if (! (directiveName == QLatin1String("pragma") || + directiveName == QLatin1String("import"))) + return false; // not a valid directive name + + // it must be a pragma or an import directive. + if (directiveName == QLatin1String("pragma")) { + // .pragma library + if (! (lex() == T_IDENTIFIER && tokenText() == QLatin1String("library"))) + return false; // expected `library + + // we found a .pragma library directive + directives->pragmaLibrary(); + + } else { + Q_ASSERT(directiveName == QLatin1String("import")); + lex(); // skip .import + + QString pathOrUri; + QString version; + bool fileImport = false; // file or uri import + + if (_tokenKind == T_STRING_LITERAL) { + // .import T_STRING_LITERAL as T_IDENTIFIER + + fileImport = true; + pathOrUri = tokenText(); + + } else if (_tokenKind == T_IDENTIFIER) { + // .import T_IDENTIFIER (. T_IDENTIFIER)* T_NUMERIC_LITERAL as T_IDENTIFIER + + pathOrUri = tokenText(); + + lex(); // skip the first T_IDENTIFIER + for (; _tokenKind == T_DOT; lex()) { + if (lex() != T_IDENTIFIER) + return false; + + pathOrUri += QLatin1Char('.'); + pathOrUri += tokenText(); + } + + if (_tokenKind != T_NUMERIC_LITERAL) + return false; // expected the module version number + + version = tokenText(); + } + + // + // recognize the mandatory `as' followed by the module name + // + if (! (lex() == T_RESERVED_WORD && tokenText() == QLatin1String("as"))) + return false; // expected `as' + + if (lex() != T_IDENTIFIER) + return false; // expected module name + + const QString module = tokenText(); + + if (fileImport) + directives->importFile(pathOrUri, module); + else + directives->importModule(pathOrUri, version, module); + } + + if (tokenStartLine() != lineNumber) + return false; // the directives cannot span over multiple lines + + // fetch the first token after the .pragma/.import directive + lex(); + } while (_tokenKind == T_DOT); + + return true; +} + #include "qmljskeywords_p.h" diff --git a/src/libs/qmljs/parser/qmljslexer_p.h b/src/libs/qmljs/parser/qmljslexer_p.h index 540df38112b..5f64ea80b84 100644 --- a/src/libs/qmljs/parser/qmljslexer_p.h +++ b/src/libs/qmljs/parser/qmljslexer_p.h @@ -54,6 +54,28 @@ namespace QmlJS { class Engine; +class QML_PARSER_EXPORT Directives { +public: + virtual ~Directives() {} + + virtual void pragmaLibrary() + { + } + + virtual void importFile(const QString &jsfile, const QString &module) + { + Q_UNUSED(jsfile); + Q_UNUSED(module); + } + + virtual void importModule(const QString &uri, const QString &version, const QString &module) + { + Q_UNUSED(uri); + Q_UNUSED(version); + Q_UNUSED(module); + } +}; + class QML_PARSER_EXPORT Lexer: public QmlJSGrammar { public: @@ -105,6 +127,12 @@ public: EqualPrefix }; + enum RegExpFlag { + RegExp_Global = 0x01, + RegExp_IgnoreCase = 0x02, + RegExp_Multiline = 0x04 + }; + public: Lexer(Engine *engine); @@ -114,10 +142,12 @@ public: int lex(); bool scanRegExp(RegExpBodyPrefix prefix = NoPrefix); + bool scanDirectives(Directives *directives); int regExpFlags() const { return _patternFlags; } QString regExpPattern() const { return _tokenText; } + int tokenKind() const; int tokenOffset() const; int tokenLength() const; @@ -135,6 +165,8 @@ public: QString errorMessage() const; bool prevTerminator() const; + bool followsClosingBrace() const; + bool canInsertAutomaticSemicolon(int token) const; enum ParenthesesState { IgnoreParentheses, @@ -192,6 +224,7 @@ private: bool _prohibitAutomaticSemicolon; bool _restrictedKeyword; bool _terminator; + bool _followsClosingBrace; bool _delimited; bool _qmlMode; }; diff --git a/src/libs/qmljs/parser/qmljsparser.cpp b/src/libs/qmljs/parser/qmljsparser.cpp index cf9ef2423f6..b85092b8da5 100644 --- a/src/libs/qmljs/parser/qmljsparser.cpp +++ b/src/libs/qmljs/parser/qmljsparser.cpp @@ -31,7 +31,7 @@ **************************************************************************/ #include -#include +#include #include @@ -67,14 +67,6 @@ void Parser::reallocateStack() string_stack = reinterpret_cast (qRealloc(string_stack, stack_size * sizeof(QStringRef))); } -inline static bool automatic(Engine *driver, int token) -{ - return token == QmlJSGrammar::T_RBRACE - || token == 0 - || driver->lexer()->prevTerminator(); -} - - Parser::Parser(Engine *engine): driver(engine), pool(engine->pool()), @@ -1687,7 +1679,7 @@ case 342: { const int errorState = state_stack[tos]; // automatic insertion of `;' - if (yytoken != -1 && t_action(errorState, T_AUTOMATIC_SEMICOLON) && automatic(driver, yytoken)) { + if (yytoken != -1 && t_action(errorState, T_AUTOMATIC_SEMICOLON) && lexer->canInsertAutomaticSemicolon(yytoken)) { SavedToken &tk = token_buffer[0]; tk.token = yytoken; tk.dval = yylval; diff --git a/src/libs/qmljs/parser/qmlutils_p.h b/src/libs/qmljs/parser/qmlutils_p.h deleted file mode 100644 index 44a08b7685e..00000000000 --- a/src/libs/qmljs/parser/qmlutils_p.h +++ /dev/null @@ -1,82 +0,0 @@ -/************************************************************************** -** -** This file is part of Qt Creator -** -** Copyright (c) 2011 Nokia Corporation and/or its subsidiary(-ies). -** -** Contact: Nokia Corporation (info@qt.nokia.com) -** -** -** GNU Lesser General Public License Usage -** -** This file may be used under the terms of the GNU Lesser General Public -** License version 2.1 as published by the Free Software Foundation and -** appearing in the file LICENSE.LGPL included in the packaging of this file. -** Please review the following information to ensure the GNU Lesser General -** Public License version 2.1 requirements will be met: -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** In addition, as a special exception, Nokia gives you certain additional -** rights. These rights are described in the Nokia Qt LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** Other Usage -** -** Alternatively, this file may be used in accordance with the terms and -** conditions contained in a signed written agreement between you and Nokia. -** -** If you have questions regarding the use of this file, please contact -** Nokia at info@qt.nokia.com. -** -**************************************************************************/ - -#ifndef QMLUTIL_P_H -#define QMLUTIL_P_H - -#include - -QT_BEGIN_NAMESPACE - -namespace QmlUtils { - -inline bool isUpper(const QChar &qc) -{ - ushort c = qc.unicode(); - return ((c >= 'A' && c <= 'Z') || (c > 127 && QChar::category(c) == QChar::Letter_Uppercase)); -} - -inline bool isLower(const QChar &qc) -{ - ushort c = qc.unicode(); - return ((c >= 'a' && c <= 'z') || (c > 127 && QChar::category(c) == QChar::Letter_Lowercase)); -} - -inline bool isLetter(const QChar &qc) -{ - ushort c = qc.unicode(); - return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c > 127 && qc.isLetter())); -} - -inline bool isDigit(const QChar &qc) -{ - ushort c = qc.unicode(); - return ((c >= '0' && c <= '9') || (c > 127 && qc.isDigit())); -} - -inline bool isLetterOrNumber(const QChar &qc) -{ - ushort c = qc.unicode(); - return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || (c > 127 && qc.isLetterOrNumber())); -} - -inline bool isSpace(const QChar &qc) -{ - ushort c = qc.unicode(); - return (c == 0x20 || (c >= 0x09 && c <= 0x0D) || c == 0x85 || (c > 127 && qc.isSpace())); -} - -} // namespace Qml - -QT_END_NAMESPACE - -#endif // QMLUTIL_P_H