QML: Allow numeric separators in numeric literals

ES2021 introduced numeric separators in numeric literals to improve
readability.

For example, one billion can be written "1_000_000_000" where "_" are
the numeric separators.

Update the QML lexer to allow numeric separators, based on
https://262.ecma-international.org/12.0/#prod-NumericLiteralSeparator.

Add a non-exhaustive test case to `tst_qqmlparser` to track the behavior.

Fixes: QTBUG-123792
Change-Id: Ie62d1f40fc8e0c7678e7dfea16408bdeeba6d150
Reviewed-by: Ulf Hermann <ulf.hermann@qt.io>
Reviewed-by: Olivier De Cannière <olivier.decanniere@qt.io>
Reviewed-by: Sami Shalayel <sami.shalayel@qt.io>
Reviewed-by: Dmitrii Akshintsev <dmitrii.akshintsev@qt.io>

1 files changed, 72 insertions, 0 deletions

diff --git a/tests/auto/qml/qqmlparser/tst_qqmlparser.cpp b/tests/auto/qml/qqmlparser/tst_qqmlparser.cpp
index eb8c6c260f..0a8411ddcf 100644
--- a/tests/auto/qml/qqmlparser/tst_qqmlparser.cpp
+++ b/tests/auto/qml/qqmlparser/tst_qqmlparser.cpp
@@ -34,6 +34,10 @@ private slots:
     void codeLocationsWithContinuationStringLiteral_data();
     void noSubstitutionTemplateLiteral();
     void templateLiteral();
+    void numericSeparator_data();
+    void numericSeparator();
+    void invalidNumericSeparator_data();
+    void invalidNumericSeparator();
     void leadingSemicolonInClass();
     void templatedReadonlyProperty();
     void qmlImportInJS();
@@ -496,6 +500,74 @@ void tst_qqmlparser::templateLiteral()
     QVERIFY(e);
 }
 
+void tst_qqmlparser::numericSeparator_data() {
+    QTest::addColumn<QString>("code");
+    QTest::addColumn<double>("expected_value");
+
+    QTest::newRow("Separator in decimal literal") << "1_000_000_000" << 1000000000.0;
+    QTest::newRow("Separator in fractional part") << "1000.22_33" << 1000.2233;
+    QTest::newRow("Separator in exponent part") << "1e1_0_0" << std::pow(10, 100);
+    QTest::newRow("Separator in positive exponent part") << "1e+1_0_0" << 1e100;
+    QTest::newRow("Separator in negative exponent part") << "1e-1_0_0" << 1e-100;
+    QTest::newRow("Separator in binary literal with b prefix") << "0b1010_0001_1000_0101" << static_cast<double>(0b1010000110000101);
+    QTest::newRow("Separator in binary literal with B prefix") << "0B01_10_01_10" << static_cast<double>(0b01100110);
+    QTest::newRow("Separator in octal literal with o prefix") << "0o1234_5670" << static_cast<double>(012345670);
+    QTest::newRow("Separator in octal literal with O prefix") << "0O7777_0000" << static_cast<double>(077770000);
+    QTest::newRow("Separator in hex literal with x prefix") << "0xA0_B0_C0" << static_cast<double>(0xA0B0C0);
+    QTest::newRow("Separator in hex literal with X prefix") << "0X1000_AAAA" << static_cast<double>(0x1000AAAA);
+}
+
+void tst_qqmlparser::numericSeparator() {
+    using namespace QQmlJS;
+
+    QFETCH(QString, code);
+    QFETCH(double, expected_value);
+
+    QQmlJS::Engine engine;
+
+    QQmlJS::Lexer lexer(&engine);
+    lexer.setCode(code, 1);
+
+    QQmlJS::Parser parser(&engine);
+    QVERIFY(parser.parseExpression());
+
+    AST::ExpressionNode *expression = parser.expression();
+    QVERIFY(expression);
+
+    auto *literal = QQmlJS::AST::cast<QQmlJS::AST::NumericLiteral *>(expression);
+    QVERIFY(literal);
+
+    QCOMPARE(literal->value, expected_value);
+    QCOMPARE(literal->firstSourceLocation().begin(), 0u);
+    QCOMPARE(literal->lastSourceLocation().end(), quint32(code.size()));
+}
+
+void tst_qqmlparser::invalidNumericSeparator_data() {
+    QTest::addColumn<QString>("code");
+    QTest::addColumn<QString>("error");
+
+    QTest::newRow("Trailing numeric separator") << "1_" << "A trailing numeric separator is not allowed in numeric literals";
+    QTest::newRow("Multiple numeric separators") << "1__2" << "There can be at most one numeric separator beetwen digits";
+}
+
+void tst_qqmlparser::invalidNumericSeparator() {
+    using namespace QQmlJS;
+
+    QFETCH(QString, code);
+    QFETCH(QString, error);
+
+    QQmlJS::Engine engine;
+
+    QQmlJS::Lexer lexer(&engine);
+    lexer.setCode(code, 1);
+
+    QQmlJS::Parser parser(&engine);
+    QVERIFY(!parser.parseExpression());
+
+    QVERIFY(lexer.errorCode() != Lexer::NoError);
+    QCOMPARE(lexer.errorMessage(), error);
+}
+
 void tst_qqmlparser::leadingSemicolonInClass()
 {
     QQmlJS::Engine engine;