From 70251c5ee7e2c96dac19e49b6ce69a55d78994a4 Mon Sep 17 00:00:00 2001 From: Lars Knoll Date: Fri, 2 Mar 2018 11:02:06 +0100 Subject: Bring JS grammar in line with ES7 spec This basically updates all grammar rules in the qqmljs.g file to be in line with the ES7 specification. Some special handling for the lookahead rules appearing in the spec was needed and is implemented through empty lookahead rules in the grammar, that might push an additional token into the token stream. Renamed some classes in the AST to be in line with the names used in ES7, and removed some other ones (SourceElements) that are no longer used. The ES7 grammar rules contain lots of variations of the base rules (with In/Return/Yield/Default suffixes). With the exception of the In and Default rules, these are implemented through state tracking in the parser and lexer. Change-Id: I4017d97cd050ed816c1dad11833e882cba30801a Reviewed-by: Simon Hausmann --- tools/qmlmin/main.cpp | 104 +++++++++++++++++++++++++++++++------------------- 1 file changed, 65 insertions(+), 39 deletions(-) (limited to 'tools') diff --git a/tools/qmlmin/main.cpp b/tools/qmlmin/main.cpp index 5641e6348e..26833d2a08 100644 --- a/tools/qmlmin/main.cpp +++ b/tools/qmlmin/main.cpp @@ -57,15 +57,48 @@ class QmlminLexer: protected Lexer, public Directives QString _fileName; QString _directives; +protected: + QVector _stateStack; + QList _tokens; + QList _tokenStrings; + int yytoken = -1; + QString yytokentext; + + void lex() { + if (_tokens.isEmpty()) { + _tokens.append(Lexer::lex()); + _tokenStrings.append(tokenText()); + } + + yytoken = _tokens.takeFirst(); + yytokentext = _tokenStrings.takeFirst(); + } + + int lookaheadToken() + { + if (yytoken < 0) + lex(); + return yytoken; + } + + void pushToken(int token) + { + _tokens.prepend(yytoken); + _tokenStrings.prepend(yytokentext); + yytoken = token; + yytokentext = QString(); + } + public: - QmlminLexer(): Lexer(&_engine) {} + QmlminLexer() + : Lexer(&_engine), _stateStack(128) {} virtual ~QmlminLexer() {} QString fileName() const { return _fileName; } bool operator()(const QString &fileName, const QString &code) { - int startToken = T_FEED_JS_PROGRAM; + int startToken = T_FEED_JS_SCRIPT; const QFileInfo fileInfo(fileName); if (fileInfo.suffix().toLower() == QLatin1String("qml")) startToken = T_FEED_UI_PROGRAM; @@ -154,6 +187,24 @@ protected: ruleno == J_SCRIPT_REGEXPLITERAL_RULE2; } + void handleLookaheads(int ruleno) { + if (ruleno == J_SCRIPT_EXPRESSIONSTATEMENTLOOKAHEAD_RULE) { + int token = lookaheadToken(); + if (token == T_LBRACE) + pushToken(T_FORCE_BLOCK); + else if (token == T_FUNCTION || token == T_CLASS || token == T_LET || token == T_CONST) + pushToken(T_FORCE_DECLARATION); + } else if (ruleno == J_SCRIPT_CONCISEBODYLOOKAHEAD_RULE) { + int token = lookaheadToken(); + if (token == T_LBRACE) + pushToken(T_FORCE_BLOCK); + } else if (ruleno == J_SCRIPT_EXPORTDECLARATIONLOOKAHEAD_RULE) { + int token = lookaheadToken(); + if (token == T_FUNCTION || token == T_CLASS) + pushToken(T_FORCE_DECLARATION); + } + } + bool scanRestOfRegExp(int ruleno, QString *restOfRegExp) { if (! scanRegExp(ruleno == J_SCRIPT_REGEXPLITERAL_RULE1 ? Lexer::NoPrefix : Lexer::EqualPrefix)) @@ -187,9 +238,6 @@ protected: class Minify: public QmlminLexer { - QVector _stateStack; - QList _tokens; - QList _tokenStrings; QString _minifiedCode; int _maxWidth; int _width; @@ -206,7 +254,7 @@ protected: }; Minify::Minify(int maxWidth) - : _stateStack(128), _maxWidth(maxWidth), _width(0) + : _maxWidth(maxWidth), _width(0) { } @@ -250,16 +298,14 @@ void Minify::escape(const QChar &ch, QString *out) bool Minify::parse(int startToken) { int yyaction = 0; - int yytoken = -1; int yytos = -1; - QString yytokentext; QString assembled; _minifiedCode.clear(); _tokens.append(startToken); _tokenStrings.append(QString()); - if (startToken == T_FEED_JS_PROGRAM) { + if (startToken == T_FEED_JS_SCRIPT) { // parse optional pragma directive DiagnosticMessage error; if (scanDirectives(this, &error)) { @@ -282,15 +328,8 @@ bool Minify::parse(int startToken) _stateStack[yytos] = yyaction; again: - if (yytoken == -1 && action_index[yyaction] != -TERMINAL_COUNT) { - if (_tokens.isEmpty()) { - _tokens.append(lex()); - _tokenStrings.append(tokenText()); - } - - yytoken = _tokens.takeFirst(); - yytokentext = _tokenStrings.takeFirst(); - } + if (yytoken == -1 && action_index[yyaction] != -TERMINAL_COUNT) + lex(); yyaction = t_action(yyaction, yytoken); if (yyaction > 0) { @@ -366,6 +405,8 @@ bool Minify::parse(int startToken) const int ruleno = -yyaction - 1; yytos -= rhs[ruleno]; + handleLookaheads(ruleno); + if (isRegExpRule(ruleno)) { QString restOfRegExp; @@ -398,13 +439,10 @@ bool Minify::parse(int startToken) class Tokenize: public QmlminLexer { - QVector _stateStack; - QList _tokens; - QList _tokenStrings; QStringList _minifiedCode; public: - Tokenize(); + Tokenize() {} QStringList tokenStream() const; @@ -412,11 +450,6 @@ protected: bool parse(int startToken) override; }; -Tokenize::Tokenize() - : _stateStack(128) -{ -} - QStringList Tokenize::tokenStream() const { return _minifiedCode; @@ -425,15 +458,13 @@ QStringList Tokenize::tokenStream() const bool Tokenize::parse(int startToken) { int yyaction = 0; - int yytoken = -1; int yytos = -1; - QString yytokentext; _minifiedCode.clear(); _tokens.append(startToken); _tokenStrings.append(QString()); - if (startToken == T_FEED_JS_PROGRAM) { + if (startToken == T_FEED_JS_SCRIPT) { // parse optional pragma directive DiagnosticMessage error; if (scanDirectives(this, &error)) { @@ -457,15 +488,8 @@ bool Tokenize::parse(int startToken) _stateStack[yytos] = yyaction; again: - if (yytoken == -1 && action_index[yyaction] != -TERMINAL_COUNT) { - if (_tokens.isEmpty()) { - _tokens.append(lex()); - _tokenStrings.append(tokenText()); - } - - yytoken = _tokens.takeFirst(); - yytokentext = _tokenStrings.takeFirst(); - } + if (yytoken == -1 && action_index[yyaction] != -TERMINAL_COUNT) + lex(); yyaction = t_action(yyaction, yytoken); if (yyaction > 0) { @@ -484,6 +508,8 @@ bool Tokenize::parse(int startToken) const int ruleno = -yyaction - 1; yytos -= rhs[ruleno]; + handleLookaheads(ruleno); + if (isRegExpRule(ruleno)) { QString restOfRegExp; -- cgit v1.2.3