diff options
author | Lars Knoll <lars.knoll@qt.io> | 2018-10-12 13:53:04 +0200 |
---|---|---|
committer | Lars Knoll <lars.knoll@qt.io> | 2018-11-02 18:43:58 +0000 |
commit | 31a9acd0348750e60feda9bdee07afc7511a6be7 (patch) | |
tree | 59299ed777b2d4bf512c08e7aa69ef72784c731d | |
parent | 5bf057eca49b59f7516c6d5bb3f4dbbeb4713942 (diff) |
Also store the raw string value for template strings in the lexer
This is required to properly implement support for tagged templates
and String.raw.
Change-Id: Id548f97dd8ebfb7206abb55967f979c3fa6f6eea
Reviewed-by: Erik Verbruggen <erik.verbruggen@qt.io>
-rw-r--r-- | src/qml/parser/qqmljslexer.cpp | 67 | ||||
-rw-r--r-- | src/qml/parser/qqmljslexer_p.h | 3 |
2 files changed, 48 insertions, 22 deletions
diff --git a/src/qml/parser/qqmljslexer.cpp b/src/qml/parser/qqmljslexer.cpp index b98bb8bac5..e5b8894fb1 100644 --- a/src/qml/parser/qqmljslexer.cpp +++ b/src/qml/parser/qqmljslexer.cpp @@ -130,6 +130,7 @@ void Lexer::setCode(const QString &code, int lineno, bool qmlMode) _tokenText.reserve(1024); _errorMessage.clear(); _tokenSpell = QStringRef(); + _rawString = QStringRef(); _codePtr = code.unicode(); _endPtr = _codePtr + code.length(); @@ -163,13 +164,20 @@ void Lexer::setCode(const QString &code, int lineno, bool qmlMode) void Lexer::scanChar() { - unsigned sequenceLength = isLineTerminatorSequence(); + if (_skipLinefeed) { + Q_ASSERT(*_codePtr == QLatin1Char('\n')); + ++_codePtr; + _skipLinefeed = false; + } _char = *_codePtr++; - if (sequenceLength == 2) - _char = *_codePtr++; - ++_currentColumnNumber; + if (isLineTerminator()) { + if (_char == QLatin1Char('\r')) { + if (_codePtr < _endPtr && *_codePtr == QLatin1Char('\n')) + _skipLinefeed = true; + _char = QLatin1Char('\n'); + } ++_currentLineNumber; _currentColumnNumber = 0; } @@ -246,6 +254,7 @@ int Lexer::lex() again: _tokenSpell = QStringRef(); + _rawString = QStringRef(); _tokenKind = scanToken(); _tokenLength = _codePtr - _tokenStartPtr - 1; @@ -821,12 +830,15 @@ int Lexer::scanString(ScanStringMode mode) QChar quote = (mode == TemplateContinuation) ? QChar(TemplateHead) : QChar(mode); bool multilineStringLiteral = false; - const QChar *startCode = _codePtr; + const QChar *startCode = _codePtr - 1; + // in case we just parsed a \r, we need to reset this flag to get things working + // correctly in the loop below and afterwards + _skipLinefeed = false; if (_engine) { while (_codePtr <= _endPtr) { - if (isLineTerminator() && quote != QLatin1Char('`')) { - if (qmlMode()) + if (isLineTerminator()) { + if ((quote == QLatin1Char('`') || qmlMode())) break; _errorCode = IllegalCharacter; _errorMessage = QCoreApplication::translate("QQmlParser", "Stray newline in string literal"); @@ -836,7 +848,8 @@ int Lexer::scanString(ScanStringMode mode) } else if (_char == '$' && quote == QLatin1Char('`')) { break; } else if (_char == quote) { - _tokenSpell = _engine->midRef(startCode - _code.unicode() - 1, _codePtr - startCode); + _tokenSpell = _engine->midRef(startCode - _code.unicode(), _codePtr - startCode - 1); + _rawString = _tokenSpell; scanChar(); if (quote == QLatin1Char('`')) @@ -849,28 +862,36 @@ int Lexer::scanString(ScanStringMode mode) else return T_STRING_LITERAL; } - scanChar(); + // don't use scanChar() here, that would transform \r sequences and the midRef() call would create the wrong result + _char = *_codePtr++; + ++_currentColumnNumber; } } + // rewind by one char, so things gets scanned correctly + --_codePtr; + _validTokenText = true; - _tokenText.resize(0); - startCode--; - while (startCode != _codePtr - 1) - _tokenText += *startCode++; + _tokenText = QString(startCode, _codePtr - startCode); + + auto setRawString = [&](const QChar *end) { + QString raw(startCode, end - startCode - 1); + raw.replace(QLatin1String("\r\n"), QLatin1String("\n")); + raw.replace(QLatin1Char('\r'), QLatin1Char('\n')); + _rawString = _engine->newStringRef(raw); + }; + + scanChar(); while (_codePtr <= _endPtr) { - if (unsigned sequenceLength = isLineTerminatorSequence()) { - multilineStringLiteral = true; - _tokenText += _char; - if (sequenceLength == 2) - _tokenText += *_codePtr; - scanChar(); - } else if (_char == mode) { + if (_char == mode) { scanChar(); - if (_engine) + if (_engine) { _tokenSpell = _engine->newStringRef(_tokenText); + if (quote == QLatin1Char('`')) + setRawString(_codePtr - 1); + } if (quote == QLatin1Char('`')) _bracesCount = _outerTemplateBraceCount.pop(); @@ -885,8 +906,10 @@ int Lexer::scanString(ScanStringMode mode) scanChar(); scanChar(); _bracesCount = 1; - if (_engine) + if (_engine) { _tokenSpell = _engine->newStringRef(_tokenText); + setRawString(_codePtr - 2); + } return (mode == TemplateHead ? T_TEMPLATE_HEAD : T_TEMPLATE_MIDDLE); } else if (_char == QLatin1Char('\\')) { diff --git a/src/qml/parser/qqmljslexer_p.h b/src/qml/parser/qqmljslexer_p.h index 644c5c09aa..03f33f6e06 100644 --- a/src/qml/parser/qqmljslexer_p.h +++ b/src/qml/parser/qqmljslexer_p.h @@ -161,6 +161,7 @@ public: int tokenStartColumn() const { return _tokenColumn; } inline QStringRef tokenSpell() const { return _tokenSpell; } + inline QStringRef rawString() const { return _rawString; } double tokenValue() const { return _tokenValue; } QString tokenText() const; @@ -213,6 +214,7 @@ private: QString _tokenText; QString _errorMessage; QStringRef _tokenSpell; + QStringRef _rawString; const QChar *_codePtr; const QChar *_endPtr; @@ -248,6 +250,7 @@ private: bool _followsClosingBrace; bool _delimited; bool _qmlMode; + bool _skipLinefeed = false; int _generatorLevel = 0; bool _staticIsKeyword = false; }; |