qqmljs.g: reduce number of conflicts

First, let the lexer create a T_FUNCTION_STAR token for "function *",
which makes the job of the parser significantly easier.

Then, by using GeneratorDeclaration instead of GeneratorExpression, we can
avoid the conflicts between UiObjectMember and PrimaryExpression

Change-Id: Ib08918e3977ed1a80cdea0948da80bf37886f7de
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Ulf Hermann <ulf.hermann@qt.io>

2 files changed, 91 insertions, 62 deletions

diff --git a/src/qml/parser/qqmljs.g b/src/qml/parser/qqmljs.g
index 098efa9344..6efe19eb21 100644
--- a/src/qml/parser/qqmljs.g
+++ b/src/qml/parser/qqmljs.g
@@ -49,6 +49,7 @@
 %token T_DIVIDE_EQ "/="         T_DO "do"                   T_DOT "."
 %token T_ELSE "else"            T_EQ "="                    T_EQ_EQ "=="
 %token T_EQ_EQ_EQ "==="         T_FINALLY "finally"         T_FOR "for"
+%token T_FUNCTION_STAR "function *"
 %token T_FUNCTION "function"    T_GE ">="                   T_GT ">"
 %token T_GT_GT ">>"             T_GT_GT_EQ ">>="            T_GT_GT_GT ">>>"
 %token T_GT_GT_GT_EQ ">>>="     T_IDENTIFIER "identifier"   T_IF "if"
@@ -1390,18 +1391,18 @@ UiObjectMember: T_READONLY T_PROPERTY UiPropertyType QmlIdentifier T_COLON Expre
     } break;
 ./
 
-UiObjectMember: FunctionDeclarationWithTypes;
+UiObjectMember: GeneratorDeclaration;
 /.
     case $rule_number: {
-        sym(1).Node = new (pool) AST::UiSourceElement(sym(1).Node);
+        auto node = new (pool) AST::UiSourceElement(sym(1).Node);
+        sym(1).Node = node;
     } break;
 ./
 
-UiObjectMember: GeneratorExpression;
+UiObjectMember: FunctionDeclarationWithTypes;
 /.
     case $rule_number: {
-        auto node = new (pool) AST::UiSourceElement(sym(1).Node);
-        sym(1).Node = node;
+        sym(1).Node = new (pool) AST::UiSourceElement(sym(1).Node);
     } break;
 ./
 
@@ -3228,7 +3229,7 @@ ExpressionStatementLookahead: ;
         int token = lookaheadToken(lexer);
         if (token == T_LBRACE)
             pushToken(T_FORCE_BLOCK);
-        else if (token == T_FUNCTION || token == T_CLASS || token == T_LET || token == T_CONST)
+        else if (token == T_FUNCTION || token == T_FUNCTION_STAR || token == T_CLASS || token == T_LET || token == T_CONST)
             pushToken(T_FORCE_DECLARATION);
     } break;
 ./
@@ -3976,27 +3977,14 @@ GeneratorRBrace: T_RBRACE;
     } break;
 ./
 
-GeneratorDeclaration: Function T_STAR BindingIdentifier GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
-/.
-    case $rule_number: {
-        AST::FunctionDeclaration *node = new (pool) AST::FunctionDeclaration(stringRef(3), sym(5).FormalParameterList, sym(8).StatementList);
-        node->functionToken = loc(1);
-        node->identifierToken = loc(3);
-        node->lparenToken = loc(4);
-        node->rparenToken = loc(6);
-        node->lbraceToken = loc(7);
-        node->rbraceToken = loc(9);
-        node->isGenerator = true;
-        sym(1).Node = node;
-    } break;
-./
+FunctionStar: T_FUNCTION_STAR %prec REDUCE_HERE;
 
-GeneratorDeclaration_Default: GeneratorDeclaration;
-GeneratorDeclaration_Default: Function T_STAR GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
+GeneratorDeclaration: FunctionStar BindingIdentifier GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
 /.
     case $rule_number: {
-        AST::FunctionDeclaration *node = new (pool) AST::FunctionDeclaration(QStringRef(), sym(4).FormalParameterList, sym(7).StatementList);
+        AST::FunctionDeclaration *node = new (pool) AST::FunctionDeclaration(stringRef(2), sym(4).FormalParameterList, sym(7).StatementList);
         node->functionToken = loc(1);
+        node->identifierToken = loc(2);
         node->lparenToken = loc(3);
         node->rparenToken = loc(5);
         node->lbraceToken = loc(6);
@@ -4006,27 +3994,28 @@ GeneratorDeclaration_Default: Function T_STAR GeneratorLParen FormalParameters T
     } break;
 ./
 
-GeneratorExpression: T_FUNCTION T_STAR BindingIdentifier GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
+GeneratorDeclaration_Default: GeneratorDeclaration;
+GeneratorDeclaration_Default: FunctionStar GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
 /.
     case $rule_number: {
-        AST::FunctionExpression *node = new (pool) AST::FunctionExpression(stringRef(3), sym(5).FormalParameterList, sym(8).StatementList);
+        AST::FunctionDeclaration *node = new (pool) AST::FunctionDeclaration(QStringRef(), sym(3).FormalParameterList, sym(6).StatementList);
         node->functionToken = loc(1);
-        if (!stringRef(3).isNull())
-          node->identifierToken = loc(3);
-        node->lparenToken = loc(4);
-        node->rparenToken = loc(6);
-        node->lbraceToken = loc(7);
-        node->rbraceToken = loc(9);
+        node->lparenToken = loc(2);
+        node->rparenToken = loc(4);
+        node->lbraceToken = loc(5);
+        node->rbraceToken = loc(7);
         node->isGenerator = true;
         sym(1).Node = node;
     } break;
 ./
 
-GeneratorExpression: T_FUNCTION T_STAR GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
+GeneratorExpression: T_FUNCTION_STAR BindingIdentifier GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
 /.
     case $rule_number: {
-        AST::FunctionExpression *node = new (pool) AST::FunctionExpression(QStringRef(), sym(4).FormalParameterList, sym(7).StatementList);
+        AST::FunctionExpression *node = new (pool) AST::FunctionExpression(stringRef(2), sym(4).FormalParameterList, sym(7).StatementList);
         node->functionToken = loc(1);
+        if (!stringRef(2).isNull())
+          node->identifierToken = loc(2);
         node->lparenToken = loc(3);
         node->rparenToken = loc(5);
         node->lbraceToken = loc(6);
@@ -4036,6 +4025,20 @@ GeneratorExpression: T_FUNCTION T_STAR GeneratorLParen FormalParameters T_RPAREN
     } break;
 ./
 
+GeneratorExpression: T_FUNCTION_STAR GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
+/.
+    case $rule_number: {
+        AST::FunctionExpression *node = new (pool) AST::FunctionExpression(QStringRef(), sym(3).FormalParameterList, sym(6).StatementList);
+        node->functionToken = loc(1);
+        node->lparenToken = loc(2);
+        node->rparenToken = loc(4);
+        node->lbraceToken = loc(5);
+        node->rbraceToken = loc(7);
+        node->isGenerator = true;
+        sym(1).Node = node;
+    } break;
+./
+
 GeneratorBody: FunctionBody;
 
 YieldExpression: T_YIELD;
@@ -4415,7 +4418,7 @@ ExportDeclarationLookahead: ;
 /.
     case $rule_number: {
         int token = lookaheadToken(lexer);
-        if (token == T_FUNCTION || token == T_CLASS)
+        if (token == T_FUNCTION || token == T_FUNCTION_STAR || token == T_CLASS)
             pushToken(T_FORCE_DECLARATION);
     } break;
 ./
diff --git a/src/qml/parser/qqmljslexer.cpp b/src/qml/parser/qqmljslexer.cpp
index 1e0ac72bd1..443e1a7476 100644
--- a/src/qml/parser/qqmljslexer.cpp
+++ b/src/qml/parser/qqmljslexer.cpp
@@ -492,6 +492,42 @@ int Lexer::scanToken()
 again:
     _validTokenText = false;
 
+    // handle comment can be called after a '/' has been read
+    // and returns true if it actually encountered a comment
+    auto handleComment = [this](){
+        if (_char == QLatin1Char('*')) {
+            scanChar();
+            while (_codePtr <= _endPtr) {
+                if (_char == QLatin1Char('*')) {
+                    scanChar();
+                    if (_char == QLatin1Char('/')) {
+                        scanChar();
+
+                        if (_engine) {
+                            _engine->addComment(tokenOffset() + 2, _codePtr - _tokenStartPtr - 1 - 4,
+                                                tokenStartLine(), tokenStartColumn() + 2);
+                        }
+
+                        return true;
+                    }
+                } else {
+                    scanChar();
+                }
+            }
+        } else if (_char == QLatin1Char('/')) {
+            while (_codePtr <= _endPtr && !isLineTerminator()) {
+                scanChar();
+            }
+            if (_engine) {
+                _engine->addComment(tokenOffset() + 2, _codePtr - _tokenStartPtr - 1 - 2,
+                                    tokenStartLine(), tokenStartColumn() + 2);
+            }
+            return true;
+        }
+        return false;
+    };
+
+
     while (_char.isSpace()) {
         if (isLineTerminator()) {
             if (_restrictedKeyword) {
@@ -599,35 +635,9 @@ again:
     case ':': return T_COLON;
 
     case '/':
-        if (_char == QLatin1Char('*')) {
-            scanChar();
-            while (_codePtr <= _endPtr) {
-                if (_char == QLatin1Char('*')) {
-                    scanChar();
-                    if (_char == QLatin1Char('/')) {
-                        scanChar();
-
-                        if (_engine) {
-                            _engine->addComment(tokenOffset() + 2, _codePtr - _tokenStartPtr - 1 - 4,
-                                                tokenStartLine(), tokenStartColumn() + 2);
-                        }
-
-                        goto again;
-                    }
-                } else {
-                    scanChar();
-                }
-            }
-        } else if (_char == QLatin1Char('/')) {
-            while (_codePtr <= _endPtr && !isLineTerminator()) {
-                scanChar();
-            }
-            if (_engine) {
-                _engine->addComment(tokenOffset() + 2, _codePtr - _tokenStartPtr - 1 - 2,
-                                    tokenStartLine(), tokenStartColumn() + 2);
-            }
+        if (handleComment())
             goto again;
-        } if (_char == QLatin1Char('=')) {
+        else if (_char == QLatin1Char('=')) {
             scanChar();
             return T_DIVIDE_EQ;
         }
@@ -829,6 +839,21 @@ again:
             if (!identifierWithEscapeChars)
                 kind = classify(_tokenStartPtr, _tokenLength, parseModeFlags());
 
+            if (kind == T_FUNCTION) {
+                continue_skipping:
+                while (_codePtr < _endPtr && _char.isSpace())
+                    scanChar();
+                if (_char == QLatin1Char('*')) {
+                    _tokenLength = _codePtr - _tokenStartPtr - 1;
+                    kind = T_FUNCTION_STAR;
+                    scanChar();
+                } else if (_char == QLatin1Char('/')) {
+                    scanChar();
+                    if (handleComment())
+                        goto continue_skipping;
+                }
+            }
+
             if (_engine) {
                 if (kind == T_IDENTIFIER && identifierWithEscapeChars)
                     _tokenSpell = _engine->newStringRef(_tokenText);
@@ -1407,6 +1432,7 @@ static const int uriTokens[] = {
     QQmlJSGrammar::T_FINALLY,
     QQmlJSGrammar::T_FOR,
     QQmlJSGrammar::T_FUNCTION,
+    QQmlJSGrammar::T_FUNCTION_STAR,
     QQmlJSGrammar::T_IF,
     QQmlJSGrammar::T_IN,
     QQmlJSGrammar::T_OF,