qmlls: Add delta support for semantic highlighting

If the document is too big, a client might not want to reperform all
highlights. Instead, they will request semanticTokens/delta to
understand what has changed on top of the latest data.

Server now holds a resultID string and the last encoded tokens. If the
client requests a delta with the same resultID then the difference is
computed and semanticeditstoken is populated. Else, servers' response will
contain the full semantic tokens.

Skip the module test for now as there is an unexpected behavior on
handling of variants in the language server api. This issue is tracked
with QTBUG-124870.

Task-number: QTBUG-120000
Change-Id: I5ea6d7a8f892d0be101dcc8f799937654618d79c
Reviewed-by: Fabian Kosmale <fabian.kosmale@qt.io>

5 files changed, 213 insertions, 2 deletions

diff --git a/tests/auto/qmlls/modules/data/highlighting/basic.qml b/tests/auto/qmlls/modules/data/highlighting/basic.qml
new file mode 100644
index 0000000000..264f553b22
--- /dev/null
+++ b/tests/auto/qmlls/modules/data/highlighting/basic.qml
@@ -0,0 +1,11 @@
+// Copyright (C) 2024 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+
+import QtQuick
+
+Item {
+    id: rootId
+    function a() {
+    
+    }
+}
diff --git a/tests/auto/qmlls/modules/tst_qmlls_modules.cpp b/tests/auto/qmlls/modules/tst_qmlls_modules.cpp
index 6b9eb87d7d..d0851f15ff 100644
--- a/tests/auto/qmlls/modules/tst_qmlls_modules.cpp
+++ b/tests/auto/qmlls/modules/tst_qmlls_modules.cpp
@@ -373,7 +373,7 @@ void tst_qmlls_modules::buildDir()
     didChange.textDocument.uri = *uri;
     didChange.textDocument.version = 2;
 
-    // change the file content to force qqmlcodemodel to recreate a new DomItem
+    // change the file content to force qqml— to recreate a new DomItem
     // if it reuses the old DomItem then it will not know about the added build directory
     TextDocumentContentChangeEvent change;
     change.range = Range{ Position{ 4, 0 }, Position{ 4, 0 } };
@@ -1609,4 +1609,88 @@ void tst_qmlls_modules::semanticHighlightingRange()
     QTRY_VERIFY_WITH_TIMEOUT(*didFinish, 10000);
 }
 
+void tst_qmlls_modules::semanticHighlightingDelta_data()
+{
+    QTest::addColumn<QString>("filePath");
+    QTest::addRow("basicDelta") << u"highlighting/basic.qml"_s;
+}
+
+void tst_qmlls_modules::semanticHighlightingDelta()
+{
+    QSKIP("This test should be skipped until QTBUG-124870 is fixed");
+    QFETCH(QString, filePath);
+    QFETCH(QString, deltaFilePath);
+
+    const auto fileItem = fileObject(testFile(filePath));
+    const auto deltaFileItem = fileObject(testFile(deltaFilePath));
+    Highlights highlights;
+    auto fullDocumentSemanticTokensData = highlights.collectTokens(fileItem, std::nullopt);
+    auto editedDocumentSemanticTokensData = highlights.collectTokens(deltaFileItem, std::nullopt);
+    const auto expectedEdits = HighlightingUtils::computeDiff(fullDocumentSemanticTokensData, editedDocumentSemanticTokensData);
+
+    const auto uri = openFile(filePath);
+    QVERIFY(uri);
+    const auto deltaUri = openFile(deltaFilePath);
+    QVERIFY(deltaUri);
+
+    std::shared_ptr<bool> didFinish = std::make_shared<bool>(false);
+    const auto cleanup = [didFinish]() { *didFinish = true; };
+
+    QLspSpecification::SemanticTokensDeltaParams params;
+    QLspSpecification::Responses::SemanticTokensDeltaResultType result;
+
+    auto &&errorHandler = [&](auto &error) {
+        QScopeGuard callAtExit(cleanup);
+        ProtocolBase::defaultResponseErrorHandler(error);
+        QVERIFY2(false, "error occurred on semantic tokens/delta");
+    };
+
+    QLspSpecification::SemanticTokensParams fullParams;
+    fullParams.textDocument.uri = *uri;
+    m_protocol->requestSemanticTokens(fullParams,
+    [&](auto res) {
+        QScopeGuard callAtExit(cleanup);
+        if (auto r = std::get_if<QLspSpecification::SemanticTokens>(&res)) {
+            params.previousResultId = r->resultId.value();
+            fullDocumentSemanticTokensData = r->data;
+        }
+    }, errorHandler);
+    QTRY_VERIFY_WITH_TIMEOUT(*didFinish, 10000);
+
+    // Change the file
+    DidChangeTextDocumentParams didChange;
+    didChange.textDocument.uri = *uri;
+    didChange.textDocument.version = 2;
+
+    TextDocumentContentChangeEvent change;
+    change.range = Range{ Position{ 8, 4 }, Position{ 8, 4 } };
+    change.text = "const Patron = 42";
+
+    didChange.contentChanges.append(change);
+    m_protocol->notifyDidChangeTextDocument(didChange);
+
+    *didFinish = false;
+    params.textDocument.uri = *uri;
+    m_protocol->requestSemanticTokensDelta(params,
+    [&](auto res) {
+        QScopeGuard callAtExit(cleanup);
+        result = res;
+    }, std::move(errorHandler));
+    QTRY_VERIFY_WITH_TIMEOUT(*didFinish, 10000);
+
+    if (const auto *const delta = std::get_if<QLspSpecification::SemanticTokensDelta>(&result)) {
+        QVERIFY(delta);
+        const auto data = delta->edits.front().data;
+        const auto start = delta->edits.front().start;
+        const auto deleteCount = delta->edits.front().deleteCount;
+        QCOMPARE(start, expectedEdits.front().start);
+        QCOMPARE(deleteCount, expectedEdits.front().deleteCount);
+        QCOMPARE(data, expectedEdits.front().data);
+    } else {
+        const auto *const full = std::get_if<QLspSpecification::SemanticTokens>(&result);
+        QVERIFY(full);
+        QCOMPARE(full->data, expectedEdits.front().data);
+    }
+}
+
 QTEST_MAIN(tst_qmlls_modules)
diff --git a/tests/auto/qmlls/modules/tst_qmlls_modules.h b/tests/auto/qmlls/modules/tst_qmlls_modules.h
index aa6263ccc6..d7b601cf5a 100644
--- a/tests/auto/qmlls/modules/tst_qmlls_modules.h
+++ b/tests/auto/qmlls/modules/tst_qmlls_modules.h
@@ -73,7 +73,8 @@ private slots:
     void semanticHighlightingFull();
     void semanticHighlightingRange_data();
     void semanticHighlightingRange();
-
+    void semanticHighlightingDelta_data();
+    void semanticHighlightingDelta();
 private:
     QProcess m_server;
     std::unique_ptr<QLanguageServerProtocol> m_protocol;
diff --git a/tests/auto/qmlls/utils/tst_qmlls_highlighting.cpp b/tests/auto/qmlls/utils/tst_qmlls_highlighting.cpp
index 004a326684..f21de11990 100644
--- a/tests/auto/qmlls/utils/tst_qmlls_highlighting.cpp
+++ b/tests/auto/qmlls/utils/tst_qmlls_highlighting.cpp
@@ -537,4 +537,114 @@ void tst_qmlls_highlighting::rangeOverlapsWithSourceLocation()
     QVERIFY(overlaps == HighlightingUtils::rangeOverlapsWithSourceLocation(sourceLocation, range));
 }
 
+void tst_qmlls_highlighting::updateResultID_data()
+{
+    QTest::addColumn<QByteArray>("currentId");
+    QTest::addColumn<QByteArray>("expectedNextId");
+
+    QTest::addRow("zero-to-one") << QByteArray("0") << QByteArray("1");
+    QTest::addRow("nine-to-ten") << QByteArray("9") << QByteArray("10");
+    QTest::addRow("nineteen-to-twenty") << QByteArray("19") << QByteArray("20");
+    QTest::addRow("twodigit-to-threedigit") << QByteArray("99") << QByteArray("100");
+}
+
+void tst_qmlls_highlighting::updateResultID()
+{
+    QFETCH(QByteArray, currentId);
+    QFETCH(QByteArray, expectedNextId);
+
+    HighlightingUtils::updateResultID(currentId);
+    QCOMPARE(currentId, expectedNextId);
+}
+
+void tst_qmlls_highlighting::computeDiff_data()
+{
+    QTest::addColumn<QList<int>>("oldData");
+    QTest::addColumn<QList<int>>("newData");
+    QTest::addColumn<QList<SemanticTokensEdit>>("expected");
+
+    {
+        QList<int> oldData { 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0};
+        QList<int> newData {  3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0};
+        SemanticTokensEdit expected;
+        expected.start = 0;
+        expected.deleteCount = 1;
+        expected.data = QList{3};
+        QTest::addRow("simple") << oldData << newData << QList{expected};
+    }
+    {
+        QList<int> oldData { 0, 0, 5, 5, 0};
+        QList<int> newData { 3, 3, 3, 3, 3, 0, 0, 5, 5, 0};
+        SemanticTokensEdit expected;
+        expected.start = 0;
+        expected.deleteCount = 0;
+        expected.data = QList{3, 3, 3, 3, 3};
+        QTest::addRow("prepend") << oldData << newData << QList{expected};
+    }
+    {
+        QList<int> oldData { 3, 3, 3, 3, 3, 0, 0, 5, 5, 0};
+        QList<int> newData { 0, 0, 5, 5, 0};
+        SemanticTokensEdit expected;
+        expected.start = 0;
+        expected.deleteCount = 5;
+        expected.data = {};
+        QTest::addRow("remove-front") << oldData << newData << QList{expected};
+    }
+    {
+        QList<int> oldData { 0, 0, 5, 5, 0};
+        QList<int> newData {  0, 0, 5, 5, 0, 1, 0, 23, 5, 0};
+        SemanticTokensEdit expected;
+        expected.start = 5;
+        expected.deleteCount = 0;
+        expected.data = QList{1, 0, 23, 5, 0};
+        QTest::addRow("append") << oldData << newData << QList{expected};
+    }
+    {
+        QList<int> oldData { 0, 0, 5, 5, 0, 1, 0, 23, 5, 0};
+        QList<int> newData { 0, 0, 5, 5, 0};
+        SemanticTokensEdit expected;
+        expected.start = 5;
+        expected.deleteCount = 5;
+        expected.data = {};
+        QTest::addRow("remove-back") << oldData << newData << QList{expected};
+    }
+    {
+        QList<int> oldData { 0, 0, 5, 5, 0, 1, 0, 23, 5, 0};
+        QList<int> newData { 0, 0, 5, 5, 0, 3, 3, 3, 3, 3, 1, 0, 23, 5, 0};
+        SemanticTokensEdit expected;
+        expected.start = 5;
+        expected.deleteCount = 0;
+        expected.data = QList{3, 3, 3, 3, 3};
+        QTest::addRow("insert-middle") << oldData << newData << QList{expected};
+    }
+    {
+        QList<int> oldData { 0, 0, 5, 5, 0, 3, 3, 3, 3, 3, 1, 0, 23, 5, 0};
+        QList<int> newData { 0, 0, 5, 5, 0, 1, 0, 23, 5, 0};
+        SemanticTokensEdit expected;
+        expected.start = 5;
+        expected.deleteCount = 5;
+        expected.data = {};
+        QTest::addRow("remove-middle") << oldData << newData << QList{expected};
+    }
+}
+
+void tst_qmlls_highlighting::computeDiff()
+{
+    QFETCH(QList<int>, oldData);
+    QFETCH(QList<int>, newData);
+    QFETCH(QList<SemanticTokensEdit>, expected);
+
+    const auto edits = HighlightingUtils::computeDiff(oldData, newData);
+    QCOMPARE(edits.size(), expected.size());
+
+    qsizetype i = 0;
+    for (const auto &edit : edits) {
+        QCOMPARE(edit.start, expected.at(i).start);
+        QCOMPARE(edit.deleteCount, expected.at(i).deleteCount);
+        QCOMPARE(edit.data, expected.at(i).data);
+        ++i;
+    }
+}
+
+
 QTEST_MAIN(tst_qmlls_highlighting)
diff --git a/tests/auto/qmlls/utils/tst_qmlls_highlighting.h b/tests/auto/qmlls/utils/tst_qmlls_highlighting.h
index f9d2b24afe..a1d0e3c9b1 100644
--- a/tests/auto/qmlls/utils/tst_qmlls_highlighting.h
+++ b/tests/auto/qmlls/utils/tst_qmlls_highlighting.h
@@ -25,6 +25,11 @@ private slots:
     void rangeOverlapsWithSourceLocation_data();
     void rangeOverlapsWithSourceLocation();
 
+    void updateResultID_data();
+    void updateResultID();
+
+    void computeDiff_data();
+    void computeDiff();
 private:
     QString m_highlightingDataDir;
 };