Also store the raw string value for template strings in the lexer

This is required to properly implement support for tagged templates
and String.raw.

Change-Id: Id548f97dd8ebfb7206abb55967f979c3fa6f6eea
Reviewed-by: Erik Verbruggen <erik.verbruggen@qt.io>

2 files changed, 48 insertions, 22 deletions

diff --git a/src/qml/parser/qqmljslexer.cpp b/src/qml/parser/qqmljslexer.cpp
index b98bb8bac5..e5b8894fb1 100644
--- a/src/qml/parser/qqmljslexer.cpp
+++ b/src/qml/parser/qqmljslexer.cpp
@@ -130,6 +130,7 @@ void Lexer::setCode(const QString &code, int lineno, bool qmlMode)
     _tokenText.reserve(1024);
     _errorMessage.clear();
     _tokenSpell = QStringRef();
+    _rawString = QStringRef();
 
     _codePtr = code.unicode();
     _endPtr = _codePtr + code.length();
@@ -163,13 +164,20 @@ void Lexer::setCode(const QString &code, int lineno, bool qmlMode)
 
 void Lexer::scanChar()
 {
-    unsigned sequenceLength = isLineTerminatorSequence();
+    if (_skipLinefeed) {
+        Q_ASSERT(*_codePtr == QLatin1Char('\n'));
+        ++_codePtr;
+        _skipLinefeed = false;
+    }
     _char = *_codePtr++;
-    if (sequenceLength == 2)
-        _char = *_codePtr++;
-
     ++_currentColumnNumber;
+
     if (isLineTerminator()) {
+        if (_char == QLatin1Char('\r')) {
+            if (_codePtr < _endPtr && *_codePtr == QLatin1Char('\n'))
+                _skipLinefeed = true;
+            _char = QLatin1Char('\n');
+        }
         ++_currentLineNumber;
         _currentColumnNumber = 0;
     }
@@ -246,6 +254,7 @@ int Lexer::lex()
 
   again:
     _tokenSpell = QStringRef();
+    _rawString = QStringRef();
     _tokenKind = scanToken();
     _tokenLength = _codePtr - _tokenStartPtr - 1;
 
@@ -821,12 +830,15 @@ int Lexer::scanString(ScanStringMode mode)
     QChar quote = (mode == TemplateContinuation) ? QChar(TemplateHead) : QChar(mode);
     bool multilineStringLiteral = false;
 
-    const QChar *startCode = _codePtr;
+    const QChar *startCode = _codePtr - 1;
+    // in case we just parsed a \r, we need to reset this flag to get things working
+    // correctly in the loop below and afterwards
+    _skipLinefeed = false;
 
     if (_engine) {
         while (_codePtr <= _endPtr) {
-            if (isLineTerminator() && quote != QLatin1Char('`')) {
-                if (qmlMode())
+            if (isLineTerminator()) {
+                if ((quote == QLatin1Char('`') || qmlMode()))
                     break;
                 _errorCode = IllegalCharacter;
                 _errorMessage = QCoreApplication::translate("QQmlParser", "Stray newline in string literal");
@@ -836,7 +848,8 @@ int Lexer::scanString(ScanStringMode mode)
             } else if (_char == '$' && quote == QLatin1Char('`')) {
                 break;
             } else if (_char == quote) {
-                _tokenSpell = _engine->midRef(startCode - _code.unicode() - 1, _codePtr - startCode);
+                _tokenSpell = _engine->midRef(startCode - _code.unicode(), _codePtr - startCode - 1);
+                _rawString = _tokenSpell;
                 scanChar();
 
                 if (quote == QLatin1Char('`'))
@@ -849,28 +862,36 @@ int Lexer::scanString(ScanStringMode mode)
                 else
                     return T_STRING_LITERAL;
             }
-            scanChar();
+            // don't use scanChar() here, that would transform \r sequences and the midRef() call would create the wrong result
+            _char = *_codePtr++;
+            ++_currentColumnNumber;
         }
     }
 
+    // rewind by one char, so things gets scanned correctly
+    --_codePtr;
+
     _validTokenText = true;
-    _tokenText.resize(0);
-    startCode--;
-    while (startCode != _codePtr - 1)
-        _tokenText += *startCode++;
+    _tokenText = QString(startCode, _codePtr - startCode);
+
+    auto setRawString = [&](const QChar *end) {
+        QString raw(startCode, end - startCode - 1);
+        raw.replace(QLatin1String("\r\n"), QLatin1String("\n"));
+        raw.replace(QLatin1Char('\r'), QLatin1Char('\n'));
+        _rawString = _engine->newStringRef(raw);
+    };
+
+    scanChar();
 
     while (_codePtr <= _endPtr) {
-        if (unsigned sequenceLength = isLineTerminatorSequence()) {
-            multilineStringLiteral = true;
-            _tokenText += _char;
-            if (sequenceLength == 2)
-                _tokenText += *_codePtr;
-            scanChar();
-        } else if (_char == mode) {
+        if (_char == mode) {
             scanChar();
 
-            if (_engine)
+            if (_engine) {
                 _tokenSpell = _engine->newStringRef(_tokenText);
+                if (quote == QLatin1Char('`'))
+                    setRawString(_codePtr - 1);
+            }
 
             if (quote == QLatin1Char('`'))
                 _bracesCount = _outerTemplateBraceCount.pop();
@@ -885,8 +906,10 @@ int Lexer::scanString(ScanStringMode mode)
             scanChar();
             scanChar();
             _bracesCount = 1;
-            if (_engine)
+            if (_engine) {
                 _tokenSpell = _engine->newStringRef(_tokenText);
+                setRawString(_codePtr - 2);
+            }
 
             return (mode == TemplateHead ? T_TEMPLATE_HEAD : T_TEMPLATE_MIDDLE);
         } else if (_char == QLatin1Char('\\')) {
diff --git a/src/qml/parser/qqmljslexer_p.h b/src/qml/parser/qqmljslexer_p.h
index 644c5c09aa..03f33f6e06 100644
--- a/src/qml/parser/qqmljslexer_p.h
+++ b/src/qml/parser/qqmljslexer_p.h
@@ -161,6 +161,7 @@ public:
     int tokenStartColumn() const { return _tokenColumn; }
 
     inline QStringRef tokenSpell() const { return _tokenSpell; }
+    inline QStringRef rawString() const { return _rawString; }
     double tokenValue() const { return _tokenValue; }
     QString tokenText() const;
 
@@ -213,6 +214,7 @@ private:
     QString _tokenText;
     QString _errorMessage;
     QStringRef _tokenSpell;
+    QStringRef _rawString;
 
     const QChar *_codePtr;
     const QChar *_endPtr;
@@ -248,6 +250,7 @@ private:
     bool _followsClosingBrace;
     bool _delimited;
     bool _qmlMode;
+    bool _skipLinefeed = false;
     int _generatorLevel = 0;
     bool _staticIsKeyword = false;
 };