qmlls: Add highlighting protocol with empty implementation

Add the non-functional semanticTokens protocol. Introduce the basic data
structures and the utility functions that is widely going to be used
during the rest of highlighting tasks.

When semanticTokens is requested, collectTokens visits the entire
domtree and accumulates the highlighting tokens. It is encoded to list
of integers as specified by the protocol at the end of collectTokens.

Add the utility tests in tst_qmlls_highlighting and module tests in
tst_qmlls_modules::semantichighlightingFull.

Task-number: QTBUG-1200000
Change-Id: I070251552a623eff4d4ff34e4201466b3ad2efab
Reviewed-by: Dmitrii Akshintsev <dmitrii.akshintsev@qt.io>
Reviewed-by: Sami Shalayel <sami.shalayel@qt.io>
Reviewed-by: Fabian Kosmale <fabian.kosmale@qt.io>

6 files changed, 537 insertions, 0 deletions

diff --git a/tests/auto/qmlls/modules/data/highlighting/bigFile.qml b/tests/auto/qmlls/modules/data/highlighting/bigFile.qml
new file mode 100644
index 0000000000..9832e8e98a
--- /dev/null
+++ b/tests/auto/qmlls/modules/data/highlighting/bigFile.qml
@@ -0,0 +1,351 @@
+// Copyright (C) 2024 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+
+import QtQuick
+
+Item {
+    id: rootId
+    function f() {
+        let sum = 0, sum2 = 0;
+        for (let i = 1; i < 42; i = i + 2) {
+            sum = sum + i;
+            {
+                let sum = 42; // another unrelated sum
+            }
+        }
+    }
+
+    readonly property int helloProperty: 0
+    property int p2: 1
+
+    function withProperty() {
+        let sum = 0, sum2 = 0;
+        for (const i = 1; i < 42; i = i + 2) {
+            sum = sum + i;
+            helloProperty = helloProperty + sum - i * p2;
+            {
+                let helloProperty = "evil";
+            }
+        }
+    }
+    Item {
+        function f() {
+            return helloProperty + p2;
+        }
+        property string helloProperty
+    }
+    component IC: Item {
+        property var helloProperty
+        function f() {
+            return helloProperty + p2;
+        }
+    }
+    component NestedComponent: Item {
+        property NestedComponent2 inner: NestedComponent2 {}
+        property int p2
+    }
+    component NestedComponent2: Item {
+        property NestedComponent3 inner
+        property int p2
+        inner: NestedComponent3 {}
+    }
+    component NestedComponent3: Item {
+        property NestedComponent4 inner
+        property int p2
+        inner: NestedComponent4 {}
+    }
+    component NestedComponent4: Item {
+        property int helloProperty
+        property int p2
+    }
+    NestedComponent {
+        id: myNested
+    }
+    function nestedUsages() {
+        let x = myNested.inner.inner.inner.helloProperty + helloProperty;
+        let a = myNested.p2 + p2;
+        let b = myNested.inner.p2 + p2;
+        let c = myNested.inner.inner.p2 + p2;
+        let d = myNested.inner.inner.inner.p2 + p2;
+    }
+
+    function recursive(n: int): int {
+        if (n > 3)
+            return 1 + recursive(recursive(x - 1) + recursive(x - 2) - recursive(x - 3));
+        else
+            return recursive(0);
+    }
+
+    property int helloRecursive: recursive(42)
+    Rectangle {
+        function f() {
+            return rootId.recursive(123);
+        }
+    }
+
+    signal helloSignal
+
+    function callSignals() {
+        helloSignal();
+        if (false) {
+            helloSignal();
+        } else {
+            // helloSignal() // not a usage btw
+            if (true)
+                helloSignal();
+        }
+    }
+    function callSignals2() {
+        helloSignal();
+        if (false) {
+            widthChanged();
+        } else {
+            // helloSignal() // not a usage btw
+            if (true)
+                widthChanged();
+            rootId.widthChanged();
+        }
+    }
+    Item {
+        function callSignalsInChild() {
+            widthChanged();
+            rootId.widthChanged();
+        }
+    }
+
+    function myHelloHandler() {
+        let x = 32;
+    }
+    onHelloSignal: myHelloHandler
+
+    property int helloPropertyBinding
+    helloPropertyBinding: 123
+
+    property int checkHandlers
+    onCheckHandlersChanged: myHelloHandler
+    onChildrenChanged: myHelloHandler
+    function callChanged() {
+        checkHandlersChanged();
+        childrenChanged();
+    }
+    property int _: 48
+    property int ______42: 48
+    property int _123a: 48
+    on_Changed: myHelloHandler
+    on______42Changed: myHelloHandler
+    on_123AChanged: myHelloHandler
+    function weirdPropertynames() {
+        _Changed();
+        ______42Changed();
+        _123aChanged();
+    }
+
+    TapHandler {
+        onTapped: myHelloHandler
+        function f() {
+            tapped();
+        }
+    }
+
+    function anotherF() {
+        helloPropertyChanged();
+    }
+    onHelloPropertyChanged: myHelloHandler
+    // Type {}
+    function foo(mouse) {
+    }
+
+    MouseArea {
+        id: area1
+        onClicked: foo
+        property int insideMouseArea1
+    }
+
+    MouseArea {
+        id: area2
+        Connections {
+            function onClicked(mouse) {
+                area1.clicked();
+                area3.clicked();
+            }
+        }
+        property int insideMouseArea2
+
+        MouseArea {
+            id: area3
+        }
+    }
+
+    property Connections c: Connections {
+        target: area3
+        onClicked: function (mouse) {}
+    }
+    function useMouseAreas() {
+        area1.clicked();
+        area2.clicked();
+        area3.clicked();
+    }
+
+    function checkParameters(a: int, b: double, {
+        x,
+        y = {},
+        z = [x, y]
+    }) {
+        return a + b + c + x + y + z;
+    }
+
+    function deconstructingUsages(xxx) {
+        let {
+            a,
+            b
+        } = xxx;
+        let c = a + b;
+    }
+
+    function k() {
+    }
+
+    function mafik() {
+        var patron = 34;
+        const upperLimit = 42;
+        do {
+            ++patron;
+            if (patron < 2)
+                continue;
+            else
+                ++patron;
+        } while (patron < upperLimit)
+        switch (patron) {
+        case 1:
+            return 23;
+        default:
+            break;
+        }
+        try {
+            {}
+        } catch (error) {
+            {}
+        } finally {}
+        for (const a in [1, 2, 3]) {
+            throw 2;
+        }
+    }
+
+    enum Test {
+        LOG
+    }
+
+    readonly property int t: 34
+    signal tt
+    required property int k
+
+    signal kkk(string a)
+    signal yyy(string a)
+
+    function ttt() {
+    }
+
+    function createComplexExpression(...objects) {
+        // Create an object that holds some data
+        let data = {
+            a: 5,
+            b: 10,
+            c: 3
+        };
+
+        // Create a complex expression using the data object
+        let expression = ((data.a + data.b * data.c) / (data.a - data.b)) ** data.c;
+
+        return expression;
+    }
+
+    function set1() {
+        const array = [1, 2, 3, 4];
+        const [a, b] = [1, 2];
+        const [aa, , bb] = array;
+        const [aaa = 23, bbb] = array;
+        const [a1, b1, ...rest1] = array;
+        const [a2, , b2, ...rest2] = array;
+        const [a3, b3, ...{
+                pop,
+                push
+            }] = array;
+        const [a4, b4, ...[c, d]] = array;
+
+        const obj = {
+            _a: 1,
+            _b: 2
+        };
+        const {
+            a5,
+            b5
+        } = obj;
+        const {
+            a6: a_,
+            b6: b1_
+        } = obj;
+        const {
+            a7: a11 = 4,
+            b11 = 34,
+            c1: b111,
+            d1
+        } = obj;
+        let key = a;
+        const {
+            [key]: a___
+        } = obj;
+    }
+
+    function set2(s: int): int {
+        // declare first
+        let a, b, a1, b1, c, d, rest, pop, push;
+        const array = [1, 2, 3, 4];
+        [a, b] = array;
+        [a, , b] = array;
+        [a = aDefault, b] = array;
+        [a, b, ...rest] = array;
+        [a, , b, ...rest] = array;
+        [a, b, ...{
+                pop,
+                push
+            }] = array;
+        [a, b, ...[c, d]] = array;
+
+        const obj = {
+            _a: 1,
+            _b: 2
+        };
+        ({
+                a,
+                b
+            } = obj); // brackets are required
+        ({
+                a: a1,
+                b: b1
+            } = obj);
+
+        const complicatedObject = {
+            a: 1,
+            b: {
+                c: 2,
+                d: {
+                    e: 3,
+                    f: [4, 5, 6]
+                }
+            },
+            g: [7, 8, 9]
+        };
+
+        const {
+            patron,
+            b: {
+                mafik,
+                d: {
+                    e,
+                    f: [, secondF, ...restF]
+                }
+            },
+            g: [firstG, ...restG]
+        } = complicatedObject;
+    }
+}
diff --git a/tests/auto/qmlls/modules/tst_qmlls_modules.cpp b/tests/auto/qmlls/modules/tst_qmlls_modules.cpp
index 9145bb7566..717b61cc28 100644
--- a/tests/auto/qmlls/modules/tst_qmlls_modules.cpp
+++ b/tests/auto/qmlls/modules/tst_qmlls_modules.cpp
@@ -3,6 +3,7 @@
 
 #include "tst_qmlls_modules.h"
 #include "QtQmlLS/private/qqmllsutils_p.h"
+#include "QtQmlLS/private/qqmlsemantictokens_p.h"
 #include <algorithm>
 #include <memory>
 #include <optional>
@@ -1494,4 +1495,68 @@ void tst_qmlls_modules::quickFixes()
     QTRY_VERIFY_WITH_TIMEOUT(codeActionOk, 5000);
 }
 
+void tst_qmlls_modules::semanticHighlightingFull_data()
+{
+    QTest::addColumn<QString>("filePath");
+    QTest::addRow("bigfile") << u"highlighting/bigFile.qml"_s;
+}
+
+void tst_qmlls_modules::semanticHighlightingFull()
+{
+    QFETCH(QString, filePath);
+
+    const auto fileObject = [](const QString &filePath){
+        QFile f(filePath);
+        DomItem file;
+        if (!f.open(QIODevice::ReadOnly))
+            return file;
+        QString code = f.readAll();
+        QQmlJS::Dom::DomCreationOptions options;
+        options.setFlag(QQmlJS::Dom::DomCreationOption::WithScriptExpressions);
+        options.setFlag(QQmlJS::Dom::DomCreationOption::WithSemanticAnalysis);
+        options.setFlag(QQmlJS::Dom::DomCreationOption::WithRecovery);
+
+        QStringList dirs = {QLibraryInfo::path(QLibraryInfo::Qml2ImportsPath)};
+        auto envPtr = QQmlJS::Dom::DomEnvironment::create(dirs,
+                QQmlJS::Dom::DomEnvironment::Option::SingleThreaded
+                        | QQmlJS::Dom::DomEnvironment::Option::NoDependencies, options);
+        envPtr->loadBuiltins();
+        envPtr->loadFile(QQmlJS::Dom::FileToLoad::fromMemory(envPtr, filePath, code),
+                         [&file](QQmlJS::Dom::Path, const QQmlJS::Dom::DomItem &, const QQmlJS::Dom::DomItem &newIt) {
+                             file = newIt.fileObject();
+                         });
+        envPtr->loadPendingDependencies();
+        return file;
+    };
+
+    const auto item = fileObject(testFile(filePath));
+    Highlights highlights;
+    const auto expectedData = highlights.collectTokens(item);
+
+    const auto uri = openFile(filePath);
+    QVERIFY(uri);
+    QLspSpecification::SemanticTokensParams params;
+    params.textDocument.uri = *uri;
+    std::shared_ptr<bool> didFinish = std::make_shared<bool>(false);
+    const auto cleanup = [didFinish]() { *didFinish = true; };
+
+    auto &&responseHandler = [&](auto res) {
+        QScopeGuard callAtExit(cleanup);
+        const auto *const result = std::get_if<QLspSpecification::SemanticTokens>(&res);
+        QVERIFY(result);
+        QList<int> data = result->data;
+        QCOMPARE(data.size(), expectedData.size());
+        QCOMPARE(data, expectedData);
+    };
+
+    auto &&errorHandler = [&](auto &error) {
+        QScopeGuard callAtExit(cleanup);
+        ProtocolBase::defaultResponseErrorHandler(error);
+        QVERIFY2(false, "error occurred on full semantic tokens");
+    };
+
+    m_protocol->requestSemanticTokens(params, std::move(responseHandler), std::move(errorHandler));
+    QTRY_VERIFY_WITH_TIMEOUT(*didFinish, 10000);
+}
+
 QTEST_MAIN(tst_qmlls_modules)
diff --git a/tests/auto/qmlls/modules/tst_qmlls_modules.h b/tests/auto/qmlls/modules/tst_qmlls_modules.h
index d48bc99407..5ffbce6c5b 100644
--- a/tests/auto/qmlls/modules/tst_qmlls_modules.h
+++ b/tests/auto/qmlls/modules/tst_qmlls_modules.h
@@ -69,6 +69,8 @@ private slots:
     void hover_data();
     void hover();
     void checkQuickSnippets();
+    void semanticHighlightingFull_data();
+    void semanticHighlightingFull();
 
 private:
     QProcess m_server;
diff --git a/tests/auto/qmlls/utils/CMakeLists.txt b/tests/auto/qmlls/utils/CMakeLists.txt
index a8a4e9f0c4..ca4a26b051 100644
--- a/tests/auto/qmlls/utils/CMakeLists.txt
+++ b/tests/auto/qmlls/utils/CMakeLists.txt
@@ -27,6 +27,21 @@ qt_internal_add_test(tst_qmlls_utils
     TESTDATA ${test_data}
 )
 
+qt_internal_add_test(tst_qmlls_highlighting
+    SOURCES
+    tst_qmlls_highlighting.cpp
+    DEFINES
+        QT_QMLLS_HIGHLIGHTS_DATADIR="${CMAKE_CURRENT_SOURCE_DIR}/data"
+    LIBRARIES
+        Qt::Core
+        Qt::QmlDomPrivate
+        Qt::LanguageServerPrivate
+        Qt::Test
+        Qt::QuickTestUtilsPrivate
+        Qt::QmlLSPrivate
+    TESTDATA ${test_data}
+)
+
 qt_internal_extend_target(tst_qmlls_utils CONDITION ANDROID OR IOS
     DEFINES
         QT_QMLLS_UTILS_DATADIR=":/domdata"
diff --git a/tests/auto/qmlls/utils/tst_qmlls_highlighting.cpp b/tests/auto/qmlls/utils/tst_qmlls_highlighting.cpp
new file mode 100644
index 0000000000..0c907b591a
--- /dev/null
+++ b/tests/auto/qmlls/utils/tst_qmlls_highlighting.cpp
@@ -0,0 +1,80 @@
+// Copyright (C) 2024 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+
+#include "tst_qmlls_highlighting.h"
+
+#include <QtQml/private/qqmljsengine_p.h>
+#include <QtQml/private/qqmljslexer_p.h>
+#include <QtQml/private/qqmljsparser_p.h>
+#include <QtQmlDom/private/qqmldomitem_p.h>
+#include <QtQmlDom/private/qqmldomtop_p.h>
+#include <QtQmlLS/private/qqmlsemantictokens_p.h>
+
+#include <QtLanguageServer/private/qlanguageserverspectypes_p.h>
+
+#include <qlist.h>
+
+using namespace QLspSpecification;
+
+tst_qmlls_highlighting::tst_qmlls_highlighting()
+    : QQmlDataTest(QT_QMLLS_HIGHLIGHTS_DATADIR) , m_highlightingDataDir(QT_QMLLS_HIGHLIGHTS_DATADIR + "/highlights"_L1)
+{
+}
+
+// Token encoding as in:
+// https://microsoft.github.io/language-server-protocol/specifications/specification-3-16/#textDocument_semanticTokens
+void tst_qmlls_highlighting::encodeSemanticTokens_data()
+{
+    QTest::addColumn<Highlights>("highlights");
+    QTest::addColumn<QList<int>>("expectedMemoryLayout");
+
+    {
+        Highlights c;
+        c.highlights().insert(0, Token());
+        QTest::addRow("empty-token-single") << c << QList {0, 0, 0, 0, 0};
+    }
+    {
+        Highlights c;
+        QQmlJS::SourceLocation loc(0, 1, 1, 1);
+        c.highlights().insert(0, Token(loc, 0, 0));
+        QTest::addRow("single-token") << c << QList {0, 0, 1, 0, 0};
+    }
+    {
+        Highlights c;
+        Token t1(QQmlJS::SourceLocation(0, 1, 1, 1), 0, 0);
+        Token t2(QQmlJS::SourceLocation(1, 1, 3, 3), 0, 0);
+        c.highlights().insert(t1.offset, t1);
+        c.highlights().insert(t2.offset, t2);
+        QTest::addRow("different-lines") << c << QList {0, 0, 1, 0, 0, 2, 2, 1, 0, 0};
+    }
+    {
+        Highlights c;
+        Token t1(QQmlJS::SourceLocation(0, 1, 1, 1), 0, 0);
+        Token t2(QQmlJS::SourceLocation(1, 1, 1, 3), 0, 0);
+        c.highlights().insert(t1.offset, t1);
+        c.highlights().insert(t2.offset, t2);
+        QTest::addRow("same-line-different-column") << c << QList {0, 0, 1, 0, 0, 0, 2, 1, 0, 0};
+    }
+    {
+        Highlights c;
+        Token t1(QQmlJS::SourceLocation(0, 1, 1, 1), 1, 0);
+        c.highlights().insert(t1.offset, t1);
+        QTest::addRow("token-type") << c << QList {0, 0, 1, 1, 0};
+    }
+    {
+        Highlights c;
+        Token t1(QQmlJS::SourceLocation(0, 1, 1, 1), 1, 1);
+        c.highlights().insert(t1.offset, t1);
+        QTest::addRow("token-modifier") << c << QList {0, 0, 1, 1, 1};
+    }
+}
+
+void tst_qmlls_highlighting::encodeSemanticTokens()
+{
+    QFETCH(Highlights, highlights);
+    QFETCH(QList<int>, expectedMemoryLayout);
+    const auto encoded = HighlightingUtils::encodeSemanticTokens(highlights);
+    QCOMPARE(encoded, expectedMemoryLayout);
+}
+
+QTEST_MAIN(tst_qmlls_highlighting)
diff --git a/tests/auto/qmlls/utils/tst_qmlls_highlighting.h b/tests/auto/qmlls/utils/tst_qmlls_highlighting.h
new file mode 100644
index 0000000000..3348660265
--- /dev/null
+++ b/tests/auto/qmlls/utils/tst_qmlls_highlighting.h
@@ -0,0 +1,24 @@
+// Copyright (C) 2024 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+
+#ifndef TST_QMLLS_HIGHLIGHTING_H
+#define TST_QMLLS_HIGHLIGHTING_H
+
+#include <QtQuickTestUtils/private/qmlutils_p.h>
+#include <QtCore/qobject.h>
+#include <QtTest/qtest.h>
+
+class tst_qmlls_highlighting : public QQmlDataTest
+{
+    Q_OBJECT
+public:
+    tst_qmlls_highlighting();
+private slots:
+    void encodeSemanticTokens_data();
+    void encodeSemanticTokens();
+
+private:
+    QString m_highlightingDataDir;
+};
+
+#endif // TST_QMLLS_HIGHLIGHTING_H