qmljs: update qmljs parser

Change-Id: I418c3f8c77f36274864edd71d8f86d37587cb3a7
Reviewed-by: Thomas Hartmann <thomas.hartmann@qt.io>
Reviewed-by: Erik Verbruggen <erik.verbruggen@qt.io>

src/libs/qmljs/parser/qmljs.g

@@ -71,12 +71,13 @@
 %token T_VAR "var"              T_VOID "void"               T_WHILE "while"
 %token T_WITH "with"            T_XOR "^"                   T_XOR_EQ "^="
 %token T_NULL "null"            T_TRUE "true"               T_FALSE "false"
-%token T_CONST "const"
+%token T_CONST "const"          T_LET "let"
 %token T_DEBUGGER "debugger"
 %token T_RESERVED_WORD "reserved word"
 %token T_MULTILINE_STRING_LITERAL "multiline string literal"
 %token T_COMMENT "comment"
 %token T_COMPATIBILITY_SEMICOLON
+%token T_ENUM "enum"
 
 --- context keywords.
 %token T_PUBLIC "public"
@@ -281,6 +282,7 @@ public:
       AST::UiArrayMemberList *UiArrayMemberList;
       AST::UiQualifiedId *UiQualifiedId;
       AST::UiQualifiedPragmaId *UiQualifiedPragmaId;
+      AST::UiEnumMemberList *UiEnumMemberList;
     };
 
 public:
@@ -449,6 +451,7 @@ Parser::Parser(Engine *engine):
     location_stack(0),
     string_stack(0),
     program(0),
+    yylval(0),
     first_token(0),
     last_token(0)
 {
@@ -1207,6 +1210,59 @@ case $rule_number: {
 }   break;
 ./
 
+UiObjectMember: T_ENUM T_IDENTIFIER T_LBRACE EnumMemberList T_RBRACE;
+/.
+case $rule_number: {
+    AST::UiEnumDeclaration *enumDeclaration = new (pool) AST::UiEnumDeclaration(stringRef(2), sym(4).UiEnumMemberList->finish());
+    enumDeclaration->enumToken = loc(1);
+    enumDeclaration->rbraceToken = loc(5);
+    sym(1).Node = enumDeclaration;
+    break;
+}
+./
+
+EnumMemberList: T_IDENTIFIER;
+/.
+case $rule_number: {
+    AST::UiEnumMemberList *node = new (pool) AST::UiEnumMemberList(stringRef(1));
+    node->memberToken = loc(1);
+    sym(1).Node = node;
+    break;
+}
+./
+
+EnumMemberList: T_IDENTIFIER T_EQ T_NUMERIC_LITERAL;
+/.
+case $rule_number: {
+    AST::UiEnumMemberList *node = new (pool) AST::UiEnumMemberList(stringRef(1), sym(3).dval);
+    node->memberToken = loc(1);
+    node->valueToken = loc(3);
+    sym(1).Node = node;
+    break;
+}
+./
+
+EnumMemberList: EnumMemberList T_COMMA T_IDENTIFIER;
+/.
+case $rule_number: {
+    AST::UiEnumMemberList *node = new (pool) AST::UiEnumMemberList(sym(1).UiEnumMemberList, stringRef(3));
+    node->memberToken = loc(3);
+    sym(1).Node = node;
+    break;
+}
+./
+
+EnumMemberList: EnumMemberList T_COMMA T_IDENTIFIER T_EQ T_NUMERIC_LITERAL;
+/.
+case $rule_number: {
+    AST::UiEnumMemberList *node = new (pool) AST::UiEnumMemberList(sym(1).UiEnumMemberList, stringRef(3), sym(5).dval);
+    node->memberToken = loc(3);
+    node->valueToken = loc(5);
+    sym(1).Node = node;
+    break;
+}
+./
+
 JsIdentifier: T_IDENTIFIER;
 
 JsIdentifier: T_PROPERTY ;
@@ -1602,6 +1658,7 @@ ReservedIdentifier: T_DEFAULT ;
 ReservedIdentifier: T_DELETE ;
 ReservedIdentifier: T_DO ;
 ReservedIdentifier: T_ELSE ;
+ReservedIdentifier: T_ENUM ;
 ReservedIdentifier: T_FALSE ;
 ReservedIdentifier: T_FINALLY ;
 ReservedIdentifier: T_FOR ;
@@ -1622,6 +1679,7 @@ ReservedIdentifier: T_VAR ;
 ReservedIdentifier: T_VOID ;
 ReservedIdentifier: T_WHILE ;
 ReservedIdentifier: T_CONST ;
+ReservedIdentifier: T_LET ;
 ReservedIdentifier: T_DEBUGGER ;
 ReservedIdentifier: T_RESERVED_WORD ;
 ReservedIdentifier: T_WITH ;
@@ -2486,14 +2544,26 @@ VariableStatement: VariableDeclarationKind VariableDeclarationList T_AUTOMATIC_S
 VariableStatement: VariableDeclarationKind VariableDeclarationList T_SEMICOLON ;
 /.
 case $rule_number: {
-  AST::VariableStatement *node = new (pool) AST::VariableStatement(
+  AST::VariableDeclaration::VariableScope s = AST::VariableDeclaration::FunctionScope;
-     sym(2).VariableDeclarationList->finish (/*readOnly=*/sym(1).ival == T_CONST));
+  if (sym(1).ival == T_LET)
+    s = AST::VariableDeclaration::BlockScope;
+  else if (sym(1).ival == T_CONST)
+    s = AST::VariableDeclaration::ReadOnlyBlockScope;
+
+  AST::VariableStatement *node = new (pool) AST::VariableStatement(sym(2).VariableDeclarationList->finish(s));
   node->declarationKindToken = loc(1);
   node->semicolonToken = loc(3);
   sym(1).Node = node;
 } break;
 ./
 
+VariableDeclarationKind: T_LET ;
+/.
+case $rule_number: {
+  sym(1).ival = T_LET;
+} break;
+./
+
 VariableDeclarationKind: T_CONST ;
 /.
 case $rule_number: {
@@ -2542,7 +2612,8 @@ case $rule_number: {
 VariableDeclaration: JsIdentifier InitialiserOpt ;
 /.
 case $rule_number: {
-  AST::VariableDeclaration *node = new (pool) AST::VariableDeclaration(stringRef(1), sym(2).Expression);
+  AST::VariableDeclaration::VariableScope s = AST::VariableDeclaration::FunctionScope;
+  AST::VariableDeclaration *node = new (pool) AST::VariableDeclaration(stringRef(1), sym(2).Expression, s);
   node->identifierToken = loc(1);
   sym(1).Node = node;
 } break;
@@ -2551,7 +2622,8 @@ case $rule_number: {
 VariableDeclarationNotIn: JsIdentifier InitialiserNotInOpt ;
 /.
 case $rule_number: {
-  AST::VariableDeclaration *node = new (pool) AST::VariableDeclaration(stringRef(1), sym(2).Expression);
+  AST::VariableDeclaration::VariableScope s = AST::VariableDeclaration::FunctionScope;
+  AST::VariableDeclaration *node = new (pool) AST::VariableDeclaration(stringRef(1), sym(2).Expression, s);
   node->identifierToken = loc(1);
   sym(1).Node = node;
 } break;
@@ -2677,8 +2749,9 @@ case $rule_number: {
 IterationStatement: T_FOR T_LPAREN T_VAR VariableDeclarationListNotIn T_SEMICOLON ExpressionOpt T_SEMICOLON ExpressionOpt T_RPAREN Statement ;
 /.
 case $rule_number: {
+  AST::VariableDeclaration::VariableScope s = AST::VariableDeclaration::FunctionScope;
   AST::LocalForStatement *node = new (pool) AST::LocalForStatement(
-     sym(4).VariableDeclarationList->finish (/*readOnly=*/false), sym(6).Expression,
+     sym(4).VariableDeclarationList->finish(s), sym(6).Expression,
      sym(8).Expression, sym(10).Statement);
   node->forToken = loc(1);
   node->lparenToken = loc(2);

src/libs/qmljs/parser/qmljsast.cpp

@@ -953,6 +953,23 @@ void UiSourceElement::accept0(Visitor *visitor)
     visitor->endVisit(this);
 }
 
+void UiEnumDeclaration::accept0(Visitor *visitor)
+{
+    if (visitor->visit(this)) {
+        accept(members, visitor);
+    }
+
+    visitor->endVisit(this);
+}
+
+void UiEnumMemberList::accept0(Visitor *visitor)
+{
+    if (visitor->visit(this)) {
+    }
+
+    visitor->endVisit(this);
+}
+
 } } // namespace QmlJS::AST
 
 QT_QML_END_NAMESPACE

src/libs/qmljs/parser/qmljsast_p.h

@@ -203,7 +203,9 @@ public:
         Kind_UiQualifiedPragmaId,
         Kind_UiScriptBinding,
         Kind_UiSourceElement,
-        Kind_UiHeaderItemList
+        Kind_UiHeaderItemList,
+        Kind_UiEnumDeclaration,
+        Kind_UiEnumMemberList
     };
 
     inline Node()
@@ -1300,10 +1302,18 @@ class QML_PARSER_EXPORT VariableDeclaration: public Node
 public:
     QMLJS_DECLARE_AST_NODE(VariableDeclaration)
 
-    VariableDeclaration(const QStringRef &n, ExpressionNode *e):
+    enum VariableScope {
-        name (n), expression (e), readOnly(false)
+        FunctionScope,
+        BlockScope, // let
+        ReadOnlyBlockScope // const
+    };
+
+    VariableDeclaration(const QStringRef &n, ExpressionNode *e, VariableScope s):
+        name (n), expression (e), scope(s)
         { kind = K; }
 
+    bool isLexicallyScoped() const { return scope != FunctionScope; }
+
     void accept0(Visitor *visitor) override;
 
     SourceLocation firstSourceLocation() const override
@@ -1315,8 +1325,8 @@ public:
 // attributes
     QStringRef name;
     ExpressionNode *expression;
-    bool readOnly;
     SourceLocation identifierToken;
+    VariableScope scope;
 };
 
 class QML_PARSER_EXPORT VariableDeclarationList: public Node
@@ -1348,14 +1358,13 @@ public:
         return declaration->lastSourceLocation();
     }
 
-    inline VariableDeclarationList *finish (bool readOnly)
+    inline VariableDeclarationList *finish(VariableDeclaration::VariableScope s)
     {
         VariableDeclarationList *front = next;
         next = 0;
-        if (readOnly) {
         VariableDeclarationList *vdl;
-            for (vdl = front; vdl != 0; vdl = vdl->next)
+        for (vdl = front; vdl != 0; vdl = vdl->next) {
-                vdl->declaration->readOnly = true;
+            vdl->declaration->scope = s;
         }
         return front;
     }
@@ -2609,7 +2618,6 @@ public:
 // attributes
     enum { Signal, Property } type;
     QStringRef typeModifier;
-    //QStringRef memberType;
     UiQualifiedId *memberType;
     QStringRef name;
     Statement *statement; // initialized with a JS expression
@@ -2774,6 +2782,81 @@ public:
     SourceLocation rbracketToken;
 };
 
+class QML_PARSER_EXPORT UiEnumMemberList: public Node
+{
+    QMLJS_DECLARE_AST_NODE(UiEnumMemberList)
+public:
+    UiEnumMemberList(const QStringRef &member, double v = 0.0)
+        : next(this), member(member), value(v)
+    { kind = K; }
+
+    UiEnumMemberList(UiEnumMemberList *previous, const QStringRef &member)
+        : member(member)
+    {
+        kind = K;
+        next = previous->next;
+        previous->next = this;
+        value = previous->value + 1;
+    }
+
+    UiEnumMemberList(UiEnumMemberList *previous, const QStringRef &member, double v)
+        : member(member), value(v)
+    {
+        kind = K;
+        next = previous->next;
+        previous->next = this;
+    }
+
+    SourceLocation firstSourceLocation() const override
+    { return memberToken; }
+
+    SourceLocation lastSourceLocation() const override
+    { return next ? next->lastSourceLocation() :
+                    valueToken.isValid() ? valueToken : memberToken; }
+
+    void accept0(Visitor *visitor) override;
+
+    UiEnumMemberList *finish()
+    {
+        UiEnumMemberList *head = next;
+        next = 0;
+        return head;
+    }
+
+// attributes
+    UiEnumMemberList *next;
+    QStringRef member;
+    double value;
+    SourceLocation memberToken;
+    SourceLocation valueToken;
+};
+
+class QML_PARSER_EXPORT UiEnumDeclaration: public UiObjectMember
+{
+public:
+    QMLJS_DECLARE_AST_NODE(UiEnumDeclaration)
+
+    UiEnumDeclaration(const QStringRef &name,
+                      UiEnumMemberList *members)
+        : name(name)
+        , members(members)
+    { kind = K; }
+
+    SourceLocation firstSourceLocation() const override
+    { return enumToken; }
+
+    SourceLocation lastSourceLocation() const override
+    { return rbraceToken; }
+
+    void accept0(Visitor *visitor) override;
+
+// attributes
+    SourceLocation enumToken;
+    SourceLocation rbraceToken;
+    QStringRef name;
+    UiEnumMemberList *members;
+};
+
 } } // namespace AST
 
 

src/libs/qmljs/parser/qmljsastfwd_p.h

@@ -166,6 +166,8 @@ class UiArrayMemberList;
 class UiQualifiedId;
 class UiQualifiedPragmaId;
 class UiHeaderItemList;
+class UiEnumDeclaration;
+class UiEnumMemberList;
 
 } } // namespace AST
 

src/libs/qmljs/parser/qmljsastvisitor_p.h

@@ -69,6 +69,8 @@ public:
     virtual bool visit(UiArrayMemberList *) { return true; }
     virtual bool visit(UiQualifiedId *) { return true; }
     virtual bool visit(UiQualifiedPragmaId *) { return true; }
+    virtual bool visit(UiEnumDeclaration *) { return true; }
+    virtual bool visit(UiEnumMemberList *) { return true; }
 
     virtual void endVisit(UiProgram *) {}
     virtual void endVisit(UiImport *) {}
@@ -86,6 +88,8 @@ public:
     virtual void endVisit(UiArrayMemberList *) {}
     virtual void endVisit(UiQualifiedId *) {}
     virtual void endVisit(UiQualifiedPragmaId *) {}
+    virtual void endVisit(UiEnumDeclaration *) {}
+    virtual void endVisit(UiEnumMemberList *) { }
 
     // QmlJS
     virtual bool visit(ThisExpression *) { return true; }

src/libs/qmljs/parser/qmljsgrammar_p.h

@@ -48,23 +48,23 @@ class QML_PARSER_EXPORT QmlJSGrammar
 public:
     enum VariousConstants {
         EOF_SYMBOL = 0,
-    REDUCE_HERE = 105,
+        REDUCE_HERE = 107,
-    SHIFT_THERE = 104,
+        SHIFT_THERE = 106,
         T_AND = 1,
         T_AND_AND = 2,
         T_AND_EQ = 3,
-    T_AS = 93,
+        T_AS = 95,
         T_AUTOMATIC_SEMICOLON = 62,
         T_BREAK = 4,
         T_CASE = 5,
         T_CATCH = 6,
         T_COLON = 7,
         T_COMMA = 8,
-    T_COMMENT = 88,
+        T_COMMENT = 89,
-    T_COMPATIBILITY_SEMICOLON = 89,
+        T_COMPATIBILITY_SEMICOLON = 90,
         T_CONST = 84,
         T_CONTINUE = 9,
-    T_DEBUGGER = 85,
+        T_DEBUGGER = 86,
         T_DEFAULT = 10,
         T_DELETE = 11,
         T_DIVIDE_ = 12,
@@ -72,22 +72,23 @@ public:
         T_DO = 14,
         T_DOT = 15,
         T_ELSE = 16,
+        T_ENUM = 91,
         T_EQ = 17,
         T_EQ_EQ = 18,
         T_EQ_EQ_EQ = 19,
-    T_ERROR = 97,
+        T_ERROR = 99,
         T_FALSE = 83,
-    T_FEED_JS_EXPRESSION = 101,
+        T_FEED_JS_EXPRESSION = 103,
-    T_FEED_JS_PROGRAM = 103,
+        T_FEED_JS_PROGRAM = 105,
-    T_FEED_JS_SOURCE_ELEMENT = 102,
+        T_FEED_JS_SOURCE_ELEMENT = 104,
-    T_FEED_JS_STATEMENT = 100,
+        T_FEED_JS_STATEMENT = 102,
-    T_FEED_UI_OBJECT_MEMBER = 99,
+        T_FEED_UI_OBJECT_MEMBER = 101,
-    T_FEED_UI_PROGRAM = 98,
+        T_FEED_UI_PROGRAM = 100,
         T_FINALLY = 20,
         T_FOR = 21,
         T_FUNCTION = 22,
         T_GE = 23,
-    T_GET = 95,
+        T_GET = 97,
         T_GT = 24,
         T_GT_GT = 25,
         T_GT_GT_EQ = 26,
@@ -95,12 +96,13 @@ public:
         T_GT_GT_GT_EQ = 28,
         T_IDENTIFIER = 29,
         T_IF = 30,
-    T_IMPORT = 91,
+        T_IMPORT = 93,
         T_IN = 31,
         T_INSTANCEOF = 32,
         T_LBRACE = 33,
         T_LBRACKET = 34,
         T_LE = 35,
+        T_LET = 85,
         T_LPAREN = 36,
         T_LT = 37,
         T_LT_LT = 38,
@@ -108,34 +110,34 @@ public:
         T_MINUS = 40,
         T_MINUS_EQ = 41,
         T_MINUS_MINUS = 42,
-    T_MULTILINE_STRING_LITERAL = 87,
+        T_MULTILINE_STRING_LITERAL = 88,
         T_NEW = 43,
         T_NOT = 44,
         T_NOT_EQ = 45,
         T_NOT_EQ_EQ = 46,
         T_NULL = 81,
         T_NUMERIC_LITERAL = 47,
-    T_ON = 94,
+        T_ON = 96,
         T_OR = 48,
         T_OR_EQ = 49,
         T_OR_OR = 50,
         T_PLUS = 51,
         T_PLUS_EQ = 52,
         T_PLUS_PLUS = 53,
-    T_PRAGMA = 92,
+        T_PRAGMA = 94,
         T_PROPERTY = 66,
-    T_PUBLIC = 90,
+        T_PUBLIC = 92,
         T_QUESTION = 54,
         T_RBRACE = 55,
         T_RBRACKET = 56,
         T_READONLY = 68,
         T_REMAINDER = 57,
         T_REMAINDER_EQ = 58,
-    T_RESERVED_WORD = 86,
+        T_RESERVED_WORD = 87,
         T_RETURN = 59,
         T_RPAREN = 60,
         T_SEMICOLON = 61,
-    T_SET = 96,
+        T_SET = 98,
         T_SIGNAL = 67,
         T_STAR = 63,
         T_STAR_EQ = 64,
@@ -154,15 +156,15 @@ public:
         T_XOR = 79,
         T_XOR_EQ = 80,
 
-    ACCEPT_STATE = 674,
+        ACCEPT_STATE = 691,
-    RULE_COUNT = 361,
+        RULE_COUNT = 369,
-    STATE_COUNT = 675,
+        STATE_COUNT = 692,
-    TERMINAL_COUNT = 106,
+        TERMINAL_COUNT = 108,
-    NON_TERMINAL_COUNT = 111,
+        NON_TERMINAL_COUNT = 112,
 
-    GOTO_INDEX_OFFSET = 675,
+        GOTO_INDEX_OFFSET = 692,
-    GOTO_INFO_OFFSET = 3078,
+        GOTO_INFO_OFFSET = 3357,
-    GOTO_CHECK_OFFSET = 3078
+        GOTO_CHECK_OFFSET = 3357
     };
 
     static const char *const     spell[];

src/libs/qmljs/parser/qmljskeywords_p.h

@@ -63,7 +63,7 @@ static inline int classify2(const QChar *s, bool qmlMode) {
   }
   else if (qmlMode && s[0].unicode() == 'o') {
     if (s[1].unicode() == 'n') {
-      return Lexer::T_ON;
+      return qmlMode ? Lexer::T_ON : Lexer::T_IDENTIFIER;
     }
   }
   return Lexer::T_IDENTIFIER;
@@ -91,6 +91,13 @@ static inline int classify3(const QChar *s, bool qmlMode) {
       }
     }
   }
+  else if (s[0].unicode() == 'l') {
+    if (s[1].unicode() == 'e') {
+      if (s[2].unicode() == 't') {
+        return int(Lexer::T_LET);
+      }
+    }
+  }
   else if (s[0].unicode() == 'n') {
     if (s[1].unicode() == 'e') {
       if (s[2].unicode() == 'w') {
@@ -159,7 +166,7 @@ static inline int classify4(const QChar *s, bool qmlMode) {
     else if (s[1].unicode() == 'n') {
       if (s[2].unicode() == 'u') {
         if (s[3].unicode() == 'm') {
-          return Lexer::T_ENUM;
+          return qmlMode ? int(Lexer::T_ENUM) : int(Lexer::T_RESERVED_WORD);
         }
       }
     }
@@ -263,7 +270,7 @@ static inline int classify5(const QChar *s, bool qmlMode) {
       if (s[2].unicode() == 'n') {
         if (s[3].unicode() == 's') {
           if (s[4].unicode() == 't') {
-            return qmlMode ? int(Lexer::T_CONST) : int(Lexer::T_RESERVED_WORD);
+            return int(Lexer::T_CONST);
           }
         }
       }

src/libs/qmljs/parser/qmljslexer.cpp

@@ -711,6 +711,11 @@ again:
                 return multilineStringLiteral ? T_MULTILINE_STRING_LITERAL : T_STRING_LITERAL;
             } else if (_char == QLatin1Char('\\')) {
                 scanChar();
+                if (_codePtr > _endPtr) {
+                    _errorCode = IllegalEscapeSequence;
+                    _errorMessage = QCoreApplication::translate("QmlParser", "End of file reached at escape sequence.");
+                    return T_ERROR;
+                }
 
                 QChar u;
 
@@ -901,7 +906,7 @@ int Lexer::scanNumber(QChar ch)
         }
     } else if (_char.isDigit() && !qmlMode()) {
         _errorCode = IllegalCharacter;
-        _errorMessage = QCoreApplication::translate("QmlParser", "Decimal numbers cannot start with \"0\".");
+        _errorMessage = QCoreApplication::translate("QmlParser", "Decimal numbers can't start with \"0\".");
         return T_ERROR;
     }
 

src/libs/qmljs/parser/qmljslexer_p.h

@@ -84,7 +84,6 @@ public:
         T_CHAR = T_RESERVED_WORD,
         T_CLASS = T_RESERVED_WORD,
         T_DOUBLE = T_RESERVED_WORD,
-        T_ENUM = T_RESERVED_WORD,
         T_EXPORT = T_RESERVED_WORD,
         T_EXTENDS = T_RESERVED_WORD,
         T_FINAL = T_RESERVED_WORD,

src/libs/qmljs/parser/qmljsparser_p.h

@@ -47,7 +47,8 @@
 //    qlalr --no-debug --no-lines --qt qmljs.g
 //
 
-#pragma once
+#ifndef QMLJSPARSER_P_H
+#define QMLJSPARSER_P_H
 
 #include "qmljsglobal_p.h"
 #include "qmljsgrammar_p.h"
@@ -110,6 +111,7 @@ public:
       AST::UiArrayMemberList *UiArrayMemberList;
       AST::UiQualifiedId *UiQualifiedId;
       AST::UiQualifiedPragmaId *UiQualifiedPragmaId;
+      AST::UiEnumMemberList *UiEnumMemberList;
     };
 
 public:
@@ -231,9 +233,12 @@ protected:
 
 
 
-#define J_SCRIPT_REGEXPLITERAL_RULE1 91
+#define J_SCRIPT_REGEXPLITERAL_RULE1 96
 
-#define J_SCRIPT_REGEXPLITERAL_RULE2 92
+#define J_SCRIPT_REGEXPLITERAL_RULE2 97
 
 QT_QML_END_NAMESPACE
 
+
+
+#endif // QMLJSPARSER_P_H