Bring JS grammar in line with ES7 spec

This basically updates all grammar rules in the
qqmljs.g file to be in line with the ES7 specification.

Some special handling for the lookahead rules appearing
in the spec was needed and is implemented through empty
lookahead rules in the grammar, that might push an
additional token into the token stream.

Renamed some classes in the AST to be in line with
the names used in ES7, and removed some other ones
(SourceElements) that are no longer used.

The ES7 grammar rules contain lots of variations of
the base rules (with In/Return/Yield/Default suffixes).
With the exception of the In and Default rules, these
are implemented through state tracking in the parser
and lexer.

Change-Id: I4017d97cd050ed816c1dad11833e882cba30801a
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>

7 files changed, 45 insertions, 129 deletions

diff --git a/src/qml/compiler/qqmlirbuilder.cpp b/src/qml/compiler/qqmlirbuilder.cpp
index 4774095a38..9de27a2588 100644
--- a/src/qml/compiler/qqmlirbuilder.cpp
+++ b/src/qml/compiler/qqmlirbuilder.cpp
@@ -1845,10 +1845,10 @@ QVector<int> JSCodeGen::generateJSCodeForFunctionsAndBindings(const QList<Compil
         else
             name = QStringLiteral("%qml-expression-entry");
 
-        QQmlJS::AST::SourceElements *body;
-        if (function)
-            body = function->body ? function->body->elements : nullptr;
-        else {
+        QQmlJS::AST::StatementList *body;
+        if (function) {
+            body = function->body;
+        } else {
             // Synthesize source elements.
             QQmlJS::MemoryPool *pool = jsEngine->pool();
 
@@ -1858,8 +1858,7 @@ QVector<int> JSCodeGen::generateJSCodeForFunctionsAndBindings(const QList<Compil
                 QQmlJS::AST::ExpressionNode *expr = node->expressionCast();
                 stmt = new (pool) QQmlJS::AST::ExpressionStatement(expr);
             }
-            QQmlJS::AST::SourceElement *element = new (pool) QQmlJS::AST::StatementSourceElement(stmt);
-            body = new (pool) QQmlJS::AST::SourceElements(element);
+            body = new (pool) QQmlJS::AST::StatementList(stmt);
             body = body->finish();
         }
 
@@ -1873,7 +1872,7 @@ QVector<int> JSCodeGen::generateJSCodeForFunctionsAndBindings(const QList<Compil
     return runtimeFunctionIndices;
 }
 
-int JSCodeGen::defineFunction(const QString &name, AST::Node *ast, AST::FormalParameterList *formals, AST::SourceElements *body)
+int JSCodeGen::defineFunction(const QString &name, AST::Node *ast, AST::FormalParameterList *formals, AST::StatementList *body)
 {
     int qmlContextTemp = -1;
     int importedScriptsTemp = -1;
diff --git a/src/qml/compiler/qqmlirbuilder_p.h b/src/qml/compiler/qqmlirbuilder_p.h
index 9845577e11..14f59dd7e8 100644
--- a/src/qml/compiler/qqmlirbuilder_p.h
+++ b/src/qml/compiler/qqmlirbuilder_p.h
@@ -621,7 +621,7 @@ struct Q_QML_PRIVATE_EXPORT JSCodeGen : public QV4::Compiler::Codegen
 
     int defineFunction(const QString &name, AST::Node *ast,
                        AST::FormalParameterList *formals,
-                       AST::SourceElements *body) override;
+                       AST::StatementList *body) override;
 
 protected:
     void beginFunctionBodyHook() override;
diff --git a/src/qml/compiler/qqmltypecompiler.cpp b/src/qml/compiler/qqmltypecompiler.cpp
index 7c8bf05ce5..34c2c3ffc9 100644
--- a/src/qml/compiler/qqmltypecompiler.cpp
+++ b/src/qml/compiler/qqmltypecompiler.cpp
@@ -488,11 +488,8 @@ bool SignalHandlerConverter::convertSignalHandlerExpressionsToFunctionDeclaratio
         }
         if (!functionDeclaration) {
             QQmlJS::AST::Statement *statement = static_cast<QQmlJS::AST::Statement*>(foe->node);
-            QQmlJS::AST::SourceElement *sourceElement = new (pool) QQmlJS::AST::StatementSourceElement(statement);
-            QQmlJS::AST::SourceElements *elements = new (pool) QQmlJS::AST::SourceElements(sourceElement);
-            elements = elements->finish();
-
-            QQmlJS::AST::FunctionBody *body = new (pool) QQmlJS::AST::FunctionBody(elements);
+            QQmlJS::AST::StatementList *body = new (pool) QQmlJS::AST::StatementList(statement);
+            body = body->finish();
 
             functionDeclaration = new (pool) QQmlJS::AST::FunctionDeclaration(compiler->newStringRef(stringAt(binding->propertyNameIndex)), paramList, body);
             functionDeclaration->lbraceToken = functionDeclaration->functionToken
diff --git a/src/qml/compiler/qv4codegen.cpp b/src/qml/compiler/qv4codegen.cpp
index 01ca3a6a5e..6c57cdba9e 100644
--- a/src/qml/compiler/qv4codegen.cpp
+++ b/src/qml/compiler/qv4codegen.cpp
@@ -125,7 +125,7 @@ void Codegen::generateFromProgram(const QString &fileName,
     if (hasError)
         return;
 
-    defineFunction(QStringLiteral("%entry"), node, nullptr, node->elements);
+    defineFunction(QStringLiteral("%entry"), node, nullptr, node->statements);
 }
 
 void Codegen::enterContext(Node *node)
@@ -342,45 +342,10 @@ Codegen::Reference Codegen::expression(ExpressionNode *ast)
     return r.result();
 }
 
-Codegen::Result Codegen::sourceElement(SourceElement *ast)
-{
-    Result r(nx);
-    if (ast) {
-        qSwap(_expr, r);
-        accept(ast);
-        qSwap(_expr, r);
-    }
-    return r;
-}
-
-void Codegen::functionBody(FunctionBody *ast)
-{
-    if (ast)
-        sourceElements(ast->elements);
-}
-
 void Codegen::program(Program *ast)
 {
     if (ast) {
-        sourceElements(ast->elements);
-    }
-}
-
-void Codegen::sourceElements(SourceElements *ast)
-{
-    bool _requiresReturnValue = false;
-    qSwap(_requiresReturnValue, requiresReturnValue);
-    for (SourceElements *it = ast; it; it = it->next) {
-        if (!it->next)
-            qSwap(_requiresReturnValue, requiresReturnValue);
-        sourceElement(it->element);
-        if (hasError)
-            return;
-        if (StatementSourceElement *sse = AST::cast<StatementSourceElement *>(it->element)) {
-            if (AST::cast<ThrowStatement *>(sse->statement) ||
-                    AST::cast<ReturnStatement *>(sse->statement))
-                return;
-        }
+        statementList(ast->statements);
     }
 }
 
@@ -394,7 +359,10 @@ void Codegen::statementList(StatementList *ast)
             it->next->statement->kind == Statement::Kind_ContinueStatement ||
             it->next->statement->kind == Statement::Kind_ReturnStatement)
                 requiresReturnValue = _requiresReturnValue;
-        statement(it->statement);
+        if (FunctionDeclaration *decl = cast<FunctionDeclaration *>(it->statement))
+            statement(decl);
+        else
+            statement(static_cast<Statement *>(it->statement));
         requiresReturnValue = false;
         if (it->statement->kind == Statement::Kind_ThrowStatement ||
             it->statement->kind == Statement::Kind_BreakStatement ||
@@ -551,19 +519,13 @@ bool Codegen::visit(FormalParameterList *)
     return false;
 }
 
-bool Codegen::visit(FunctionBody *)
-{
-    Q_UNREACHABLE();
-    return false;
-}
-
 bool Codegen::visit(Program *)
 {
     Q_UNREACHABLE();
     return false;
 }
 
-bool Codegen::visit(PropertyAssignmentList *)
+bool Codegen::visit(PropertyDefinitionList *)
 {
     Q_UNREACHABLE();
     return false;
@@ -581,12 +543,6 @@ bool Codegen::visit(PropertyGetterSetter *)
     return false;
 }
 
-bool Codegen::visit(SourceElements *)
-{
-    Q_UNREACHABLE();
-    return false;
-}
-
 bool Codegen::visit(StatementList *)
 {
     Q_UNREACHABLE();
@@ -1672,7 +1628,7 @@ bool Codegen::visit(FunctionExpression *ast)
 
     RegisterScope scope(this);
 
-    int function = defineFunction(ast->name.toString(), ast, ast->formals, ast->body ? ast->body->elements : nullptr);
+    int function = defineFunction(ast->name.toString(), ast, ast->formals, ast->body);
     loadClosure(function);
     _expr.setResult(Reference::fromAccumulator(this));
     return false;
@@ -1880,7 +1836,7 @@ bool Codegen::visit(ObjectLiteral *ast)
 
     RegisterScope scope(this);
 
-    for (PropertyAssignmentList *it = ast->properties; it; it = it->next) {
+    for (PropertyDefinitionList *it = ast->properties; it; it = it->next) {
         QString name = it->assignment->name->asString();
         if (PropertyNameAndValue *nv = AST::cast<AST::PropertyNameAndValue *>(it->assignment)) {
             Reference value = expression(nv->value);
@@ -1896,7 +1852,7 @@ bool Codegen::visit(ObjectLiteral *ast)
 
             v.rvalue = value.storeOnStack();
         } else if (PropertyGetterSetter *gs = AST::cast<AST::PropertyGetterSetter *>(it->assignment)) {
-            const int function = defineFunction(name, gs, gs->formals, gs->functionBody ? gs->functionBody->elements : nullptr);
+            const int function = defineFunction(name, gs, gs->formals, gs->functionBody);
             ObjectPropertyValue &v = valueMap[name];
             if (v.rvalue.isValid() ||
                 (gs->type == PropertyGetterSetter::Getter && v.hasGetter()) ||
@@ -2256,14 +2212,7 @@ static bool endsWithReturn(Node *node)
     if (AST::cast<ThrowStatement *>(node))
         return true;
     if (Program *p = AST::cast<Program *>(node))
-        return endsWithReturn(p->elements);
-    if (SourceElements *se = AST::cast<SourceElements *>(node)) {
-        while (se->next)
-            se = se->next;
-        return endsWithReturn(se->element);
-    }
-    if (StatementSourceElement *sse = AST::cast<StatementSourceElement *>(node))
-        return endsWithReturn(sse->statement);
+        return endsWithReturn(p->statements);
     if (StatementList *sl = AST::cast<StatementList *>(node)) {
         while (sl->next)
             sl = sl->next;
@@ -2278,7 +2227,7 @@ static bool endsWithReturn(Node *node)
 
 int Codegen::defineFunction(const QString &name, AST::Node *ast,
                             AST::FormalParameterList *formals,
-                            AST::SourceElements *body)
+                            AST::StatementList *body)
 {
     enterContext(ast);
 
@@ -2367,7 +2316,7 @@ int Codegen::defineFunction(const QString &name, AST::Node *ast,
     for (const Context::Member &member : qAsConst(_context->members)) {
         if (member.function) {
             const int function = defineFunction(member.function->name.toString(), member.function, member.function->formals,
-                                                member.function->body ? member.function->body->elements : nullptr);
+                                                member.function->body);
             loadClosure(function);
             if (! _context->parent) {
                 Reference::fromName(this, member.function->name.toString()).storeConsumeAccumulator();
@@ -2416,7 +2365,7 @@ int Codegen::defineFunction(const QString &name, AST::Node *ast,
 
     beginFunctionBodyHook();
 
-    sourceElements(body);
+    statementList(body);
 
     if (hasError || !endsWithReturn(body)) {
         bytecodeGenerator->setLocation(ast->lastSourceLocation());
@@ -2450,24 +2399,6 @@ int Codegen::defineFunction(const QString &name, AST::Node *ast,
     return leaveContext();
 }
 
-bool Codegen::visit(FunctionSourceElement *ast)
-{
-    if (hasError)
-        return false;
-
-    statement(ast->declaration);
-    return false;
-}
-
-bool Codegen::visit(StatementSourceElement *ast)
-{
-    if (hasError)
-        return false;
-
-    statement(ast->statement);
-    return false;
-}
-
 bool Codegen::visit(Block *ast)
 {
     if (hasError)
diff --git a/src/qml/compiler/qv4codegen_p.h b/src/qml/compiler/qv4codegen_p.h
index e708d56478..5ff0aa5c30 100644
--- a/src/qml/compiler/qv4codegen_p.h
+++ b/src/qml/compiler/qv4codegen_p.h
@@ -495,7 +495,7 @@ protected:
     // Returns index in _module->functions
     virtual int defineFunction(const QString &name, AST::Node *ast,
                                AST::FormalParameterList *formals,
-                               AST::SourceElements *body);
+                               AST::StatementList *body);
 
     void statement(AST::Statement *ast);
     void statement(AST::ExpressionNode *ast);
@@ -503,13 +503,10 @@ protected:
                    const BytecodeGenerator::Label *iffalse,
                    bool trueBlockFollowsCondition);
     Reference expression(AST::ExpressionNode *ast);
-    Result sourceElement(AST::SourceElement *ast);
 
     void accept(AST::Node *node);
 
-    void functionBody(AST::FunctionBody *ast);
     void program(AST::Program *ast);
-    void sourceElements(AST::SourceElements *ast);
     void statementList(AST::StatementList *ast);
     void variableDeclaration(AST::VariableDeclaration *ast);
     void variableDeclarationList(AST::VariableDeclarationList *ast);
@@ -537,12 +534,10 @@ protected:
     bool visit(AST::Elision *ast) override;
     bool visit(AST::Finally *ast) override;
     bool visit(AST::FormalParameterList *ast) override;
-    bool visit(AST::FunctionBody *ast) override;
     bool visit(AST::Program *ast) override;
     bool visit(AST::PropertyNameAndValue *ast) override;
-    bool visit(AST::PropertyAssignmentList *ast) override;
+    bool visit(AST::PropertyDefinitionList *ast) override;
     bool visit(AST::PropertyGetterSetter *ast) override;
-    bool visit(AST::SourceElements *ast) override;
     bool visit(AST::StatementList *ast) override;
     bool visit(AST::UiArrayMemberList *ast) override;
     bool visit(AST::UiImport *ast) override;
@@ -593,10 +588,6 @@ protected:
     bool visit(AST::VoidExpression *ast) override;
     bool visit(AST::FunctionDeclaration *ast) override;
 
-    // source elements
-    bool visit(AST::FunctionSourceElement *ast) override;
-    bool visit(AST::StatementSourceElement *ast) override;
-
     // statements
     bool visit(AST::Block *ast) override;
     bool visit(AST::BreakStatement *ast) override;
diff --git a/src/qml/compiler/qv4compilerscanfunctions.cpp b/src/qml/compiler/qv4compilerscanfunctions.cpp
index c732e60b34..5b7103323e 100644
--- a/src/qml/compiler/qv4compilerscanfunctions.cpp
+++ b/src/qml/compiler/qv4compilerscanfunctions.cpp
@@ -95,25 +95,23 @@ void ScanFunctions::leaveEnvironment()
     _context = _contextStack.isEmpty() ? 0 : _contextStack.top();
 }
 
-void ScanFunctions::checkDirectivePrologue(SourceElements *ast)
+void ScanFunctions::checkDirectivePrologue(StatementList *ast)
 {
-    for (SourceElements *it = ast; it; it = it->next) {
-        if (StatementSourceElement *stmt = cast<StatementSourceElement *>(it->element)) {
-            if (ExpressionStatement *expr = cast<ExpressionStatement *>(stmt->statement)) {
-                if (StringLiteral *strLit = cast<StringLiteral *>(expr->expression)) {
-                    // Use the source code, because the StringLiteral's
-                    // value might have escape sequences in it, which is not
-                    // allowed.
-                    if (strLit->literalToken.length < 2)
-                        continue;
-                    QStringRef str = _sourceCode.midRef(strLit->literalToken.offset + 1, strLit->literalToken.length - 2);
-                    if (str == QLatin1String("use strict")) {
-                        _context->isStrict = true;
-                    } else {
-                        // TODO: give a warning.
-                    }
+    for (StatementList *it = ast; it; it = it->next) {
+        if (ExpressionStatement *expr = cast<ExpressionStatement *>(it->statement)) {
+            if (StringLiteral *strLit = cast<StringLiteral *>(expr->expression)) {
+                // Use the source code, because the StringLiteral's
+                // value might have escape sequences in it, which is not
+                // allowed.
+                if (strLit->literalToken.length < 2)
                     continue;
+                QStringRef str = _sourceCode.midRef(strLit->literalToken.offset + 1, strLit->literalToken.length - 2);
+                if (str == QLatin1String("use strict")) {
+                    _context->isStrict = true;
+                } else {
+                    // TODO: give a warning.
                 }
+                continue;
             }
         }
 
@@ -141,7 +139,7 @@ void ScanFunctions::checkName(const QStringRef &name, const SourceLocation &loc)
 bool ScanFunctions::visit(Program *ast)
 {
     enterEnvironment(ast, defaultProgramMode);
-    checkDirectivePrologue(ast->elements);
+    checkDirectivePrologue(ast->statements);
     return true;
 }
 
@@ -273,7 +271,7 @@ void ScanFunctions::endVisit(FunctionExpression *)
 bool ScanFunctions::visit(ObjectLiteral *ast)
 {
     int argc = 0;
-    for (PropertyAssignmentList *it = ast->properties; it; it = it->next) {
+    for (PropertyDefinitionList *it = ast->properties; it; it = it->next) {
         QString key = it->assignment->name->asString();
         if (QV4::String::toArrayIndex(key) != UINT_MAX)
             ++argc;
@@ -392,7 +390,7 @@ bool ScanFunctions::visit(Block *ast) {
     return false;
 }
 
-void ScanFunctions::enterFunction(Node *ast, const QString &name, FormalParameterList *formals, FunctionBody *body, FunctionExpression *expr)
+void ScanFunctions::enterFunction(Node *ast, const QString &name, FormalParameterList *formals, StatementList *body, FunctionExpression *expr)
 {
     Context *outerContext = _context;
     enterEnvironment(ast, FunctionCode);
@@ -419,7 +417,7 @@ void ScanFunctions::enterFunction(Node *ast, const QString &name, FormalParamete
     _context->formals = formals;
 
     if (body && !_context->isStrict)
-        checkDirectivePrologue(body->elements);
+        checkDirectivePrologue(body);
 
     bool isSimpleParameterList = formals->isSimpleParameterList();
 
diff --git a/src/qml/compiler/qv4compilerscanfunctions_p.h b/src/qml/compiler/qv4compilerscanfunctions_p.h
index 0767b25094..940bd9f73a 100644
--- a/src/qml/compiler/qv4compilerscanfunctions_p.h
+++ b/src/qml/compiler/qv4compilerscanfunctions_p.h
@@ -95,7 +95,7 @@ protected:
     using Visitor::visit;
     using Visitor::endVisit;
 
-    void checkDirectivePrologue(AST::SourceElements *ast);
+    void checkDirectivePrologue(AST::StatementList *ast);
 
     void checkName(const QStringRef &name, const AST::SourceLocation &loc);
 
@@ -136,7 +136,7 @@ protected:
     bool visit(AST::Block *ast) override;
 
 protected:
-    void enterFunction(AST::Node *ast, const QString &name, AST::FormalParameterList *formals, AST::FunctionBody *body, AST::FunctionExpression *expr);
+    void enterFunction(AST::Node *ast, const QString &name, AST::FormalParameterList *formals, AST::StatementList *body, AST::FunctionExpression *expr);
 
     void calcEscapingVariables();
 // fields: