Added support for generator functions and yield expressions to the AST

Some smaller changes to the codegen are included as well to ensure
that we catch all uses of generators and properly throw an unimplemented
error on them for now.

Change-Id: Ib915a0e862e128644ff00dfe989507783c912c66
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>

13 files changed, 167 insertions, 18 deletions

diff --git a/src/qml/compiler/qv4codegen.cpp b/src/qml/compiler/qv4codegen.cpp
index 0033383189..d72162eeb1 100644
--- a/src/qml/compiler/qv4codegen.cpp
+++ b/src/qml/compiler/qv4codegen.cpp
@@ -2260,6 +2260,12 @@ bool Codegen::visit(FunctionDeclaration * ast)
     return false;
 }
 
+bool Codegen::visit(YieldExpression *ast)
+{
+    throwSyntaxError(ast->firstSourceLocation(), QLatin1String("Support for 'yield' unimplemented."));
+    return false;
+}
+
 static bool endsWithReturn(Node *node)
 {
     if (!node)
@@ -2288,6 +2294,10 @@ int Codegen::defineFunction(const QString &name, AST::Node *ast,
 {
     enterContext(ast);
 
+    if (_context->isGenerator) {
+        throwSyntaxError(ast->firstSourceLocation(), QLatin1String("Support for generator functions unimplemented."));
+    }
+
     if (_context->functionIndex >= 0)
         // already defined
         return leaveContext();
diff --git a/src/qml/compiler/qv4codegen_p.h b/src/qml/compiler/qv4codegen_p.h
index 6de6affdcc..140f804d4a 100644
--- a/src/qml/compiler/qv4codegen_p.h
+++ b/src/qml/compiler/qv4codegen_p.h
@@ -588,6 +588,7 @@ protected:
     bool visit(AST::UnaryPlusExpression *ast) override;
     bool visit(AST::VoidExpression *ast) override;
     bool visit(AST::FunctionDeclaration *ast) override;
+    bool visit(AST::YieldExpression *ast) override;
 
     // statements
     bool visit(AST::Block *ast) override;
diff --git a/src/qml/compiler/qv4compileddata_p.h b/src/qml/compiler/qv4compileddata_p.h
index 48bef10c6d..8f49fbd728 100644
--- a/src/qml/compiler/qv4compileddata_p.h
+++ b/src/qml/compiler/qv4compileddata_p.h
@@ -211,6 +211,7 @@ struct Function
         HasDirectEval       = 0x2,
         UsesArgumentsObject = 0x4,
         IsArrowFunction     = 0x8,
+        IsGenerator         = 0x20,
         HasCatchOrWith      = 0x10
     };
 
diff --git a/src/qml/compiler/qv4compiler.cpp b/src/qml/compiler/qv4compiler.cpp
index 4bc3940a02..d870c34101 100644
--- a/src/qml/compiler/qv4compiler.cpp
+++ b/src/qml/compiler/qv4compiler.cpp
@@ -308,6 +308,8 @@ void QV4::Compiler::JSUnitGenerator::writeFunction(char *f, QV4::Compiler::Conte
         function->flags |= CompiledData::Function::IsStrict;
     if (irFunction->isArrowFunction)
         function->flags |= CompiledData::Function::IsArrowFunction;
+    if (irFunction->isGenerator)
+        function->flags |= CompiledData::Function::IsGenerator;
     if (irFunction->hasTry || irFunction->hasWith)
         function->flags |= CompiledData::Function::HasCatchOrWith;
     function->nestedFunctionIndex =
diff --git a/src/qml/compiler/qv4compilercontext_p.h b/src/qml/compiler/qv4compilercontext_p.h
index 89bdfc9d8e..9ce0d5fa1e 100644
--- a/src/qml/compiler/qv4compilercontext_p.h
+++ b/src/qml/compiler/qv4compilercontext_p.h
@@ -142,6 +142,7 @@ struct Context {
     bool hasNestedFunctions = false;
     bool isStrict = false;
     bool isArrowFunction = false;
+    bool isGenerator = false;
     bool usesThis = false;
     bool hasTry = false;
     bool hasWith = false;
diff --git a/src/qml/compiler/qv4compilerscanfunctions.cpp b/src/qml/compiler/qv4compilerscanfunctions.cpp
index 5c606e7f7f..57f9105484 100644
--- a/src/qml/compiler/qv4compilerscanfunctions.cpp
+++ b/src/qml/compiler/qv4compilerscanfunctions.cpp
@@ -260,7 +260,7 @@ bool ScanFunctions::enterFunction(FunctionExpression *ast, bool enterName)
 {
     if (_context->isStrict && (ast->name == QLatin1String("eval") || ast->name == QLatin1String("arguments")))
         _cg->throwSyntaxError(ast->identifierToken, QStringLiteral("Function name may not be eval or arguments in strict mode"));
-    return enterFunction(ast, ast->name.toString(), ast->formals, ast->body, enterName ? ast : nullptr);
+    return enterFunction(ast, ast->name.toString(), ast->formals, ast->body, enterName);
 }
 
 void ScanFunctions::endVisit(FunctionExpression *)
@@ -289,7 +289,7 @@ bool ScanFunctions::visit(ObjectLiteral *ast)
 bool ScanFunctions::visit(PropertyGetterSetter *ast)
 {
     TemporaryBoolAssignment allowFuncDecls(_allowFuncDecls, true);
-    return enterFunction(ast, QString(), ast->formals, ast->functionBody, /*FunctionExpression*/nullptr);
+    return enterFunction(ast, QString(), ast->formals, ast->functionBody, /*enterName */ false);
 }
 
 void ScanFunctions::endVisit(PropertyGetterSetter *)
@@ -388,19 +388,23 @@ bool ScanFunctions::visit(Block *ast) {
     return false;
 }
 
-bool ScanFunctions::enterFunction(Node *ast, const QString &name, FormalParameterList *formals, StatementList *body, FunctionExpression *expr)
+bool ScanFunctions::enterFunction(Node *ast, const QString &name, FormalParameterList *formals, StatementList *body, bool enterName)
 {
     Context *outerContext = _context;
     enterEnvironment(ast, FunctionCode);
 
+    FunctionExpression *expr = AST::cast<FunctionExpression *>(ast);
+    if (!expr)
+        expr = AST::cast<FunctionDeclaration *>(ast);
     if (outerContext) {
         outerContext->hasNestedFunctions = true;
         // The identifier of a function expression cannot be referenced from the enclosing environment.
-        if (expr) {
-            if (!outerContext->addLocalVar(name, Context::FunctionDefinition, AST::VariableDeclaration::FunctionScope, expr)) {
+        if (enterName) {
+            if (!outerContext->addLocalVar(name, Context::FunctionDefinition, AST::VariableScope::Var, expr)) {
                 _cg->throwSyntaxError(ast->firstSourceLocation(), QStringLiteral("Identifier %1 has already been declared").arg(name));
                 return false;
             }
+            outerContext->addLocalVar(name, Context::FunctionDefinition, AST::VariableScope::Var, expr);
         }
         if (name == QLatin1String("arguments"))
             outerContext->usesArgumentsObject = Context::ArgumentsObjectNotUsed;
@@ -408,8 +412,12 @@ bool ScanFunctions::enterFunction(Node *ast, const QString &name, FormalParamete
 
     if (formals->containsName(QStringLiteral("arguments")))
         _context->usesArgumentsObject = Context::ArgumentsObjectNotUsed;
-    if (expr && expr->isArrowFunction)
-        _context->isArrowFunction = true;
+    if (expr) {
+        if (expr->isArrowFunction)
+            _context->isArrowFunction = true;
+        else if (expr->isGenerator)
+            _context->isGenerator = true;
+    }
 
 
     if (!name.isEmpty() && !formals->containsName(name))
diff --git a/src/qml/compiler/qv4compilerscanfunctions_p.h b/src/qml/compiler/qv4compilerscanfunctions_p.h
index cf79da2203..43fcc36897 100644
--- a/src/qml/compiler/qv4compilerscanfunctions_p.h
+++ b/src/qml/compiler/qv4compilerscanfunctions_p.h
@@ -136,7 +136,7 @@ protected:
     bool visit(AST::Block *ast) override;
 
 protected:
-    bool enterFunction(AST::Node *ast, const QString &name, AST::FormalParameterList *formals, AST::StatementList *body, AST::FunctionExpression *expr);
+    bool enterFunction(AST::Node *ast, const QString &name, AST::FormalParameterList *formals, AST::StatementList *body, bool enterName);
 
     void calcEscapingVariables();
 // fields:
diff --git a/src/qml/parser/qqmljs.g b/src/qml/parser/qqmljs.g
index b9697a726f..22b123e6e0 100644
--- a/src/qml/parser/qqmljs.g
+++ b/src/qml/parser/qqmljs.g
@@ -3597,8 +3597,22 @@ MethodDefinition: PropertyName T_LPAREN StrictFormalParameters T_RPAREN Function
     } break;
 ./
 
-MethodDefinition: GeneratorMethod;
-/.  case $rule_number: { UNIMPLEMENTED; } ./
+MethodDefinition: T_STAR PropertyName T_LPAREN StrictFormalParameters T_RPAREN GeneratorLBrace GeneratorBody GeneratorRBrace;
+/.
+    case $rule_number: {
+        AST::FunctionExpression *f = new (pool) AST::FunctionExpression(stringRef(2), sym(4).FormalParameterList, sym(7).StatementList);
+        f->functionToken = loc(1);
+        f->lparenToken = loc(3);
+        f->rparenToken = loc(5);
+        f->lbraceToken = loc(6);
+        f->rbraceToken = loc(8);
+        f->isGenerator = true;
+        AST::PropertyNameAndValue *node = new (pool) AST::PropertyNameAndValue(sym(2).PropertyName, f);
+        node->colonToken = loc(2);
+        sym(1).Node = node;
+    } break;
+./
+
 
 MethodDefinition: T_GET PropertyName T_LPAREN T_RPAREN FunctionLBrace FunctionBody FunctionRBrace;
 /.
@@ -3652,27 +3666,101 @@ GeneratorRBrace: T_RBRACE;
     } break;
 ./
 
-GeneratorMethod: T_STAR PropertyName T_LPAREN StrictFormalParameters T_RPAREN GeneratorLBrace GeneratorBody GeneratorRBrace;
-/.  case $rule_number: { UNIMPLEMENTED; } ./
-
 GeneratorDeclaration: T_FUNCTION T_STAR BindingIdentifier T_LPAREN FormalParameters T_RPAREN GeneratorLBrace GeneratorBody GeneratorRBrace;
-/.  case $rule_number: { UNIMPLEMENTED; } ./
+/.
+    case $rule_number: {
+        AST::FunctionDeclaration *node = new (pool) AST::FunctionDeclaration(stringRef(3), sym(5).FormalParameterList, sym(8).StatementList);
+        node->functionToken = loc(1);
+        node->identifierToken = loc(3);
+        node->lparenToken = loc(4);
+        node->rparenToken = loc(6);
+        node->lbraceToken = loc(7);
+        node->rbraceToken = loc(9);
+        node->isGenerator = true;
+        sym(1).Node = node;
+    } break;
+./
 
 GeneratorDeclaration_Default: GeneratorDeclaration;
 GeneratorDeclaration_Default: T_FUNCTION T_STAR T_LPAREN FormalParameters T_RPAREN GeneratorLBrace GeneratorBody GeneratorRBrace;
-/.  case $rule_number: { UNIMPLEMENTED; } ./
+/.
+    case $rule_number: {
+        AST::FunctionDeclaration *node = new (pool) AST::FunctionDeclaration(stringRef(1), sym(4).FormalParameterList, sym(7).StatementList);
+        node->functionToken = loc(1);
+        node->identifierToken = loc(1);
+        node->lparenToken = loc(3);
+        node->rparenToken = loc(5);
+        node->lbraceToken = loc(6);
+        node->rbraceToken = loc(8);
+        node->isGenerator = true;
+        sym(1).Node = node;
+    } break;
+./
 
 GeneratorExpression: T_FUNCTION T_STAR BindingIdentifier T_LPAREN FormalParameters T_RPAREN GeneratorLBrace GeneratorBody GeneratorRBrace;
-/.  case $rule_number: { UNIMPLEMENTED; } ./
+/.
+    case $rule_number: {
+        AST::FunctionExpression *node = new (pool) AST::FunctionExpression(stringRef(3), sym(5).FormalParameterList, sym(8).StatementList);
+        node->functionToken = loc(1);
+        if (!stringRef(3).isNull())
+          node->identifierToken = loc(3);
+        node->lparenToken = loc(4);
+        node->rparenToken = loc(6);
+        node->lbraceToken = loc(7);
+        node->rbraceToken = loc(9);
+        node->isGenerator = true;
+        sym(1).Node = node;
+    } break;
+./
+
 GeneratorExpression: T_FUNCTION T_STAR T_LPAREN FormalParameters T_RPAREN GeneratorLBrace GeneratorBody GeneratorRBrace;
-/.  case $rule_number: { UNIMPLEMENTED; } ./
+/.
+    case $rule_number: {
+        AST::FunctionExpression *node = new (pool) AST::FunctionExpression(QStringRef(), sym(4).FormalParameterList, sym(7).StatementList);
+        node->functionToken = loc(1);
+        node->lparenToken = loc(3);
+        node->rparenToken = loc(5);
+        node->lbraceToken = loc(6);
+        node->rbraceToken = loc(8);
+        node->isGenerator = true;
+        sym(1).Node = node;
+    } break;
+./
 
 GeneratorBody: FunctionBody;
 
+YieldExpression: T_YIELD T_AUTOMATIC_SEMICOLON;
+YieldExpression: T_YIELD T_SEMICOLON;
+/.
+    case $rule_number: {
+        AST::YieldExpression *node = new (pool) AST::YieldExpression();
+        node->yieldToken = loc(1);
+        sym(1).Node = node;
+    } break;
+./
+
 YieldExpression: T_YIELD T_STAR AssignmentExpression;
+/.  case $rule_number: Q_FALLTHROUGH(); ./
 YieldExpression_In: T_YIELD T_STAR AssignmentExpression_In;
+/.
+    case $rule_number: {
+        AST::YieldExpression *node = new (pool) AST::YieldExpression(sym(3).Expression);
+        node->yieldToken = loc(1);
+        node->isYieldStar = true;
+        sym(1).Node = node;
+    } break;
+./
+
 YieldExpression: T_YIELD AssignmentExpression;
+/.  case $rule_number: Q_FALLTHROUGH(); ./
 YieldExpression_In: T_YIELD AssignmentExpression_In;
+/.
+    case $rule_number: {
+        AST::YieldExpression *node = new (pool) AST::YieldExpression(sym(2).Expression);
+        node->yieldToken = loc(1);
+        sym(1).Node = node;
+    } break;
+./
 
 
 ClassDeclaration: T_CLASS BindingIdentifier ClassTail;
diff --git a/src/qml/parser/qqmljsast.cpp b/src/qml/parser/qqmljsast.cpp
index 829887ab9d..35dd84a161 100644
--- a/src/qml/parser/qqmljsast.cpp
+++ b/src/qml/parser/qqmljsast.cpp
@@ -643,6 +643,16 @@ void ReturnStatement::accept0(Visitor *visitor)
     visitor->endVisit(this);
 }
 
+void YieldExpression::accept0(Visitor *visitor)
+{
+    if (visitor->visit(this)) {
+        accept(expression, visitor);
+    }
+
+    visitor->endVisit(this);
+}
+
+
 void WithStatement::accept0(Visitor *visitor)
 {
     if (visitor->visit(this)) {
@@ -1117,7 +1127,6 @@ void ComputedPropertyName::accept0(Visitor *visitor)
     }
 
     visitor->endVisit(this);
-
 }
 
 } } // namespace QQmlJS::AST
diff --git a/src/qml/parser/qqmljsast_p.h b/src/qml/parser/qqmljsast_p.h
index 21e27de5ac..e88eee4a37 100644
--- a/src/qml/parser/qqmljsast_p.h
+++ b/src/qml/parser/qqmljsast_p.h
@@ -170,6 +170,7 @@ public:
         Kind_NewMemberExpression,
         Kind_NotExpression,
         Kind_NullExpression,
+        Kind_YieldExpression,
         Kind_NumericLiteral,
         Kind_NumericLiteralPropertyName,
         Kind_ObjectLiteral,
@@ -1801,6 +1802,28 @@ public:
     SourceLocation semicolonToken;
 };
 
+class QML_PARSER_EXPORT YieldExpression: public ExpressionNode
+{
+public:
+    QQMLJS_DECLARE_AST_NODE(YieldExpression)
+
+    YieldExpression(ExpressionNode *e = nullptr):
+        expression (e) { kind = K; }
+
+    void accept0(Visitor *visitor) override;
+
+    SourceLocation firstSourceLocation() const override
+    { return yieldToken; }
+
+    SourceLocation lastSourceLocation() const override
+    { return expression ? expression->lastSourceLocation() : yieldToken; }
+
+// attributes
+    ExpressionNode *expression;
+    bool isYieldStar = false;
+    SourceLocation yieldToken;
+};
+
 class QML_PARSER_EXPORT WithStatement: public Statement
 {
 public:
@@ -2113,6 +2136,7 @@ public:
 // attributes
     QStringRef name;
     bool isArrowFunction = false;
+    bool isGenerator = false;
     FormalParameterList *formals;
     StatementList *body;
     SourceLocation functionToken;
diff --git a/src/qml/parser/qqmljsastfwd_p.h b/src/qml/parser/qqmljsastfwd_p.h
index 1bf8ce0fb8..6fc32a44f0 100644
--- a/src/qml/parser/qqmljsastfwd_p.h
+++ b/src/qml/parser/qqmljsastfwd_p.h
@@ -127,6 +127,7 @@ class NotExpression;
 class BinaryExpression;
 class ConditionalExpression;
 class Expression; // ### rename
+class YieldExpression;
 class Block;
 class StatementList;
 class VariableStatement;
diff --git a/src/qml/parser/qqmljsastvisitor_p.h b/src/qml/parser/qqmljsastvisitor_p.h
index d995ce6f0b..20117b2687 100644
--- a/src/qml/parser/qqmljsastvisitor_p.h
+++ b/src/qml/parser/qqmljsastvisitor_p.h
@@ -287,6 +287,9 @@ public:
     virtual bool visit(ReturnStatement *) { return true; }
     virtual void endVisit(ReturnStatement *) {}
 
+    virtual bool visit(YieldExpression *) { return true; }
+    virtual void endVisit(YieldExpression *) {}
+
     virtual bool visit(WithStatement *) { return true; }
     virtual void endVisit(WithStatement *) {}
 
diff --git a/src/qml/parser/qqmljslexer.cpp b/src/qml/parser/qqmljslexer.cpp
index 453c8ca474..3dbe535bcf 100644
--- a/src/qml/parser/qqmljslexer.cpp
+++ b/src/qml/parser/qqmljslexer.cpp
@@ -284,6 +284,7 @@ int Lexer::lex()
     case T_CONTINUE:
     case T_BREAK:
     case T_RETURN:
+    case T_YIELD:
     case T_THROW:
         _restrictedKeyword = true;
         break;