Merge remote-tracking branch 'origin/5.12' into dev

Change-Id: I85be5e6ef173fa733d7febd3209164bb31762e0d

4 files changed, 68 insertions, 26 deletions

diff --git a/src/qml/parser/qqmljs.g b/src/qml/parser/qqmljs.g
index a7ae664d72..6549e5bfa3 100644
--- a/src/qml/parser/qqmljs.g
+++ b/src/qml/parser/qqmljs.g
@@ -392,6 +392,9 @@ protected:
     inline QStringRef &stringRef(int index)
     { return string_stack [tos + index - 1]; }
 
+    inline QStringRef &rawStringRef(int index)
+    { return rawString_stack [tos + index - 1]; }
+
     inline AST::SourceLocation &loc(int index)
     { return location_stack [tos + index - 1]; }
 
@@ -416,6 +419,7 @@ protected:
     int *state_stack = nullptr;
     AST::SourceLocation *location_stack = nullptr;
     QVector<QStringRef> string_stack;
+    QVector<QStringRef> rawString_stack;
 
     AST::Node *program = nullptr;
 
@@ -427,11 +431,13 @@ protected:
        double dval;
        AST::SourceLocation loc;
        QStringRef spell;
+       QStringRef raw;
     };
 
     int yytoken = -1;
     double yylval = 0.;
     QStringRef yytokenspell;
+    QStringRef yytokenraw;
     AST::SourceLocation yylloc;
     AST::SourceLocation yyprevlloc;
 
@@ -493,6 +499,7 @@ void Parser::reallocateStack()
     state_stack = reinterpret_cast<int*> (realloc(state_stack, stack_size * sizeof(int)));
     location_stack = reinterpret_cast<AST::SourceLocation*> (realloc(location_stack, stack_size * sizeof(AST::SourceLocation)));
     string_stack.resize(stack_size);
+    rawString_stack.resize(stack_size);
 }
 
 Parser::Parser(Engine *engine):
@@ -555,6 +562,7 @@ void Parser::pushToken(int token)
     last_token->token = yytoken;
     last_token->dval = yylval;
     last_token->spell = yytokenspell;
+    last_token->raw = yytokenraw;
     last_token->loc = yylloc;
     ++last_token;
     yytoken = token;
@@ -566,6 +574,7 @@ int Parser::lookaheadToken(Lexer *lexer)
         yytoken = lexer->lex();
         yylval = lexer->tokenValue();
         yytokenspell = lexer->tokenSpell();
+        yytokenraw = lexer->rawString();
         yylloc = location(lexer);
     }
     return yytoken;
@@ -618,11 +627,13 @@ bool Parser::parse(int startToken)
                 yytoken = lexer->lex();
                 yylval = lexer->tokenValue();
                 yytokenspell = lexer->tokenSpell();
+                yytokenraw = lexer->rawString();
                 yylloc = location(lexer);
             } else {
                 yytoken = first_token->token;
                 yylval = first_token->dval;
                 yytokenspell = first_token->spell;
+                yytokenraw = first_token->raw;
                 yylloc = first_token->loc;
                 ++first_token;
                 if (first_token == last_token)
@@ -643,6 +654,7 @@ bool Parser::parse(int startToken)
                 yytoken = -1;
                 sym(1).dval = yylval;
                 stringRef(1) = yytokenspell;
+                rawStringRef(1) = yytokenraw;
                 loc(1) = yylloc;
             } else {
               --tos;
@@ -1889,7 +1901,7 @@ TemplateLiteral: T_NO_SUBSTITUTION_TEMPLATE;
 TemplateSpans: T_TEMPLATE_TAIL;
 /.
     case $rule_number: {
-        AST::TemplateLiteral *node = new (pool) AST::TemplateLiteral(stringRef(1), nullptr);
+        AST::TemplateLiteral *node = new (pool) AST::TemplateLiteral(stringRef(1), rawStringRef(1), nullptr);
         node->literalToken = loc(1);
         sym(1).Node = node;
     } break;
@@ -1901,7 +1913,7 @@ TemplateSpans: T_TEMPLATE_MIDDLE Expression TemplateSpans;
 TemplateLiteral: T_TEMPLATE_HEAD Expression TemplateSpans;
 /.
     case $rule_number: {
-        AST::TemplateLiteral *node = new (pool) AST::TemplateLiteral(stringRef(1), sym(2).Expression);
+        AST::TemplateLiteral *node = new (pool) AST::TemplateLiteral(stringRef(1), rawStringRef(1), sym(2).Expression);
         node->next = sym(3).Template;
         node->literalToken = loc(1);
         sym(1).Node = node;
@@ -4354,6 +4366,7 @@ ExportSpecifier: IdentifierName T_AS IdentifierName;
             tk.token = yytoken;
             tk.dval = yylval;
             tk.spell = yytokenspell;
+            tk.raw = yytokenraw;
             tk.loc = yylloc;
 
             yylloc = yyprevlloc;
@@ -4380,11 +4393,13 @@ ExportSpecifier: IdentifierName T_AS IdentifierName;
         token_buffer[0].token = yytoken;
         token_buffer[0].dval = yylval;
         token_buffer[0].spell = yytokenspell;
+        token_buffer[0].raw = yytokenraw;
         token_buffer[0].loc = yylloc;
 
         token_buffer[1].token = yytoken       = lexer->lex();
         token_buffer[1].dval  = yylval        = lexer->tokenValue();
         token_buffer[1].spell = yytokenspell  = lexer->tokenSpell();
+        token_buffer[1].raw   = yytokenraw    = lexer->rawString();
         token_buffer[1].loc   = yylloc        = location(lexer);
 
         if (t_action(errorState, yytoken)) {
diff --git a/src/qml/parser/qqmljsast_p.h b/src/qml/parser/qqmljsast_p.h
index 72c47cbe32..43aeec6525 100644
--- a/src/qml/parser/qqmljsast_p.h
+++ b/src/qml/parser/qqmljsast_p.h
@@ -500,8 +500,8 @@ class QML_PARSER_EXPORT TemplateLiteral : public LeftHandSideExpression
 public:
     QQMLJS_DECLARE_AST_NODE(TemplateLiteral)
 
-    TemplateLiteral(const QStringRef &str, ExpressionNode *e)
-        : value(str), expression(e), next(nullptr)
+    TemplateLiteral(const QStringRef &str, const QStringRef &raw, ExpressionNode *e)
+        : value(str), rawValue(raw), expression(e), next(nullptr)
     { kind = K; }
 
     SourceLocation firstSourceLocation() const override
@@ -513,6 +513,7 @@ public:
     void accept0(Visitor *visitor) override;
 
     QStringRef value;
+    QStringRef rawValue;
     ExpressionNode *expression;
     TemplateLiteral *next;
     SourceLocation literalToken;
diff --git a/src/qml/parser/qqmljslexer.cpp b/src/qml/parser/qqmljslexer.cpp
index b98bb8bac5..e5b8894fb1 100644
--- a/src/qml/parser/qqmljslexer.cpp
+++ b/src/qml/parser/qqmljslexer.cpp
@@ -130,6 +130,7 @@ void Lexer::setCode(const QString &code, int lineno, bool qmlMode)
     _tokenText.reserve(1024);
     _errorMessage.clear();
     _tokenSpell = QStringRef();
+    _rawString = QStringRef();
 
     _codePtr = code.unicode();
     _endPtr = _codePtr + code.length();
@@ -163,13 +164,20 @@ void Lexer::setCode(const QString &code, int lineno, bool qmlMode)
 
 void Lexer::scanChar()
 {
-    unsigned sequenceLength = isLineTerminatorSequence();
+    if (_skipLinefeed) {
+        Q_ASSERT(*_codePtr == QLatin1Char('\n'));
+        ++_codePtr;
+        _skipLinefeed = false;
+    }
     _char = *_codePtr++;
-    if (sequenceLength == 2)
-        _char = *_codePtr++;
-
     ++_currentColumnNumber;
+
     if (isLineTerminator()) {
+        if (_char == QLatin1Char('\r')) {
+            if (_codePtr < _endPtr && *_codePtr == QLatin1Char('\n'))
+                _skipLinefeed = true;
+            _char = QLatin1Char('\n');
+        }
         ++_currentLineNumber;
         _currentColumnNumber = 0;
     }
@@ -246,6 +254,7 @@ int Lexer::lex()
 
   again:
     _tokenSpell = QStringRef();
+    _rawString = QStringRef();
     _tokenKind = scanToken();
     _tokenLength = _codePtr - _tokenStartPtr - 1;
 
@@ -821,12 +830,15 @@ int Lexer::scanString(ScanStringMode mode)
     QChar quote = (mode == TemplateContinuation) ? QChar(TemplateHead) : QChar(mode);
     bool multilineStringLiteral = false;
 
-    const QChar *startCode = _codePtr;
+    const QChar *startCode = _codePtr - 1;
+    // in case we just parsed a \r, we need to reset this flag to get things working
+    // correctly in the loop below and afterwards
+    _skipLinefeed = false;
 
     if (_engine) {
         while (_codePtr <= _endPtr) {
-            if (isLineTerminator() && quote != QLatin1Char('`')) {
-                if (qmlMode())
+            if (isLineTerminator()) {
+                if ((quote == QLatin1Char('`') || qmlMode()))
                     break;
                 _errorCode = IllegalCharacter;
                 _errorMessage = QCoreApplication::translate("QQmlParser", "Stray newline in string literal");
@@ -836,7 +848,8 @@ int Lexer::scanString(ScanStringMode mode)
             } else if (_char == '$' && quote == QLatin1Char('`')) {
                 break;
             } else if (_char == quote) {
-                _tokenSpell = _engine->midRef(startCode - _code.unicode() - 1, _codePtr - startCode);
+                _tokenSpell = _engine->midRef(startCode - _code.unicode(), _codePtr - startCode - 1);
+                _rawString = _tokenSpell;
                 scanChar();
 
                 if (quote == QLatin1Char('`'))
@@ -849,28 +862,36 @@ int Lexer::scanString(ScanStringMode mode)
                 else
                     return T_STRING_LITERAL;
             }
-            scanChar();
+            // don't use scanChar() here, that would transform \r sequences and the midRef() call would create the wrong result
+            _char = *_codePtr++;
+            ++_currentColumnNumber;
         }
     }
 
+    // rewind by one char, so things gets scanned correctly
+    --_codePtr;
+
     _validTokenText = true;
-    _tokenText.resize(0);
-    startCode--;
-    while (startCode != _codePtr - 1)
-        _tokenText += *startCode++;
+    _tokenText = QString(startCode, _codePtr - startCode);
+
+    auto setRawString = [&](const QChar *end) {
+        QString raw(startCode, end - startCode - 1);
+        raw.replace(QLatin1String("\r\n"), QLatin1String("\n"));
+        raw.replace(QLatin1Char('\r'), QLatin1Char('\n'));
+        _rawString = _engine->newStringRef(raw);
+    };
+
+    scanChar();
 
     while (_codePtr <= _endPtr) {
-        if (unsigned sequenceLength = isLineTerminatorSequence()) {
-            multilineStringLiteral = true;
-            _tokenText += _char;
-            if (sequenceLength == 2)
-                _tokenText += *_codePtr;
-            scanChar();
-        } else if (_char == mode) {
+        if (_char == mode) {
             scanChar();
 
-            if (_engine)
+            if (_engine) {
                 _tokenSpell = _engine->newStringRef(_tokenText);
+                if (quote == QLatin1Char('`'))
+                    setRawString(_codePtr - 1);
+            }
 
             if (quote == QLatin1Char('`'))
                 _bracesCount = _outerTemplateBraceCount.pop();
@@ -885,8 +906,10 @@ int Lexer::scanString(ScanStringMode mode)
             scanChar();
             scanChar();
             _bracesCount = 1;
-            if (_engine)
+            if (_engine) {
                 _tokenSpell = _engine->newStringRef(_tokenText);
+                setRawString(_codePtr - 2);
+            }
 
             return (mode == TemplateHead ? T_TEMPLATE_HEAD : T_TEMPLATE_MIDDLE);
         } else if (_char == QLatin1Char('\\')) {
diff --git a/src/qml/parser/qqmljslexer_p.h b/src/qml/parser/qqmljslexer_p.h
index 644c5c09aa..03f33f6e06 100644
--- a/src/qml/parser/qqmljslexer_p.h
+++ b/src/qml/parser/qqmljslexer_p.h
@@ -161,6 +161,7 @@ public:
     int tokenStartColumn() const { return _tokenColumn; }
 
     inline QStringRef tokenSpell() const { return _tokenSpell; }
+    inline QStringRef rawString() const { return _rawString; }
     double tokenValue() const { return _tokenValue; }
     QString tokenText() const;
 
@@ -213,6 +214,7 @@ private:
     QString _tokenText;
     QString _errorMessage;
     QStringRef _tokenSpell;
+    QStringRef _rawString;
 
     const QChar *_codePtr;
     const QChar *_endPtr;
@@ -248,6 +250,7 @@ private:
     bool _followsClosingBrace;
     bool _delimited;
     bool _qmlMode;
+    bool _skipLinefeed = false;
     int _generatorLevel = 0;
     bool _staticIsKeyword = false;
 };