semantichighlighting: add highlights for comments

Comments can be laying on multiple lines. Usually, lsp clients will not
support multiline tokens. Therefore, introduce a utility function that
splits the highlighting of multiline tokens into multiple highlighted
tokens.

Task-number: QTBUG-1200000
Change-Id: Ib81e2781e39c3bf952b17d788031384ccab91e7b
Reviewed-by: Dmitrii Akshintsev <dmitrii.akshintsev@qt.io>
Reviewed-by: Fabian Kosmale <fabian.kosmale@qt.io>
Reviewed-by: Sami Shalayel <sami.shalayel@qt.io>

3 files changed, 161 insertions, 0 deletions

diff --git a/tests/auto/qmlls/utils/data/highlights/comments.qml b/tests/auto/qmlls/utils/data/highlights/comments.qml
new file mode 100644
index 0000000000..351aaee36c
--- /dev/null
+++ b/tests/auto/qmlls/utils/data/highlights/comments.qml
@@ -0,0 +1,19 @@
+// Copyright (C) 2024 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+import QtQuick
+Item {
+/*
+    multiline comment
+*/
+
+/* single line comment    */
+// another
+
+    function inc() {
+        // in
+
+        /*
+            inside js
+        */
+    }
+}
diff --git a/tests/auto/qmlls/utils/tst_qmlls_highlighting.cpp b/tests/auto/qmlls/utils/tst_qmlls_highlighting.cpp
index 0c907b591a..778b513368 100644
--- a/tests/auto/qmlls/utils/tst_qmlls_highlighting.cpp
+++ b/tests/auto/qmlls/utils/tst_qmlls_highlighting.cpp
@@ -77,4 +77,141 @@ void tst_qmlls_highlighting::encodeSemanticTokens()
     QCOMPARE(encoded, expectedMemoryLayout);
 }
 
+struct LineLength
+{
+    quint32 startLine;
+    quint32 length;
+};
+
+void tst_qmlls_highlighting::sourceLocationsFromMultiLineToken_data()
+{
+    QTest::addColumn<QString>("source");
+    QTest::addColumn<QList<LineLength>>("expectedLines");
+
+    QTest::addRow("multilineComment1") << R"("line 1
+line 2
+line 3 ")" << QList{ LineLength{ 1, 7 }, LineLength{ 2, 6 }, LineLength{ 3, 8 } };
+
+    QTest::addRow("prePostNewlines") <<
+            R"("
+
+")" << QList{ LineLength{ 1, 1 }, LineLength{ 2, 0 }, LineLength{ 3, 1 } };
+    QTest::addRow("windows-newline")
+            << QString::fromUtf8("\"test\r\nwindows\r\nnewline\"")
+            << QList{ LineLength{ 1, 5 }, LineLength{ 2, 7 }, LineLength{ 3, 8 } };
+}
+
+void tst_qmlls_highlighting::sourceLocationsFromMultiLineToken()
+{
+    QFETCH(QString, source);
+    QFETCH(QList<LineLength>, expectedLines);
+    using namespace QQmlJS::AST;
+
+    QQmlJS::Engine jsEngine;
+    QQmlJS::Lexer lexer(&jsEngine);
+    lexer.setCode(source, 1, true);
+    QQmlJS::Parser parser(&jsEngine);
+    parser.parseExpression();
+    const auto expression = parser.expression();
+
+    auto *literal = QQmlJS::AST::cast<QQmlJS::AST::StringLiteral *>(expression);
+    const auto locs =
+            HighlightingUtils::sourceLocationsFromMultiLineToken(source, literal->literalToken);
+
+    [&]() {
+        QCOMPARE(locs.size(), expectedLines.size());
+
+        for (auto i = 0; i < locs.size(); ++i) {
+            QCOMPARE(locs[i].startLine, expectedLines[i].startLine);
+            QCOMPARE(locs[i].length, expectedLines[i].length);
+        }
+    }();
+
+    if (QTest::currentTestFailed()) {
+
+        qDebug() << "Actual locations";
+        for (auto i = 0; i < locs.size(); ++i) {
+            qDebug() << "Startline :" << locs[i].startLine << "Length " << locs[i].length;
+        }
+
+        qDebug() << "Expected locations";
+        for (auto i = 0; i < expectedLines.size(); ++i) {
+            qDebug() << "Startline :" << expectedLines[i].startLine
+                     << "Length :" << expectedLines[i].length;
+        }
+    }
+}
+
+void tst_qmlls_highlighting::highlights_data()
+{
+    using namespace QQmlJS::Dom;
+    QTest::addColumn<DomItem>("fileItem");
+    QTest::addColumn<Token>("expectedHighlightedToken");
+
+    const auto fileObject = [](const QString &filePath){
+        QFile f(filePath);
+        DomItem file;
+        if (!f.open(QIODevice::ReadOnly | QIODevice::Text))
+            return file;
+        QString code = f.readAll();
+        auto envPtr = DomEnvironment::create(
+                QStringList(),
+                QQmlJS::Dom::DomEnvironment::Option::SingleThreaded
+                        | QQmlJS::Dom::DomEnvironment::Option::NoDependencies);
+        envPtr->loadFile(FileToLoad::fromMemory(envPtr, filePath, code),
+                         [&file](Path, const DomItem &, const DomItem &newIt) {
+                             file = newIt.fileObject();
+                         });
+        envPtr->loadPendingDependencies();
+        return file;
+    };
+
+    { // Comments
+        const auto filePath = m_highlightingDataDir + "/comments.qml";
+        const auto fileItem = fileObject(filePath);
+        // Copyright (C) 2023 The Qt Company Ltd.
+        QTest::addRow("single-line-1")
+                << fileItem
+                << Token(QQmlJS::SourceLocation(0, 41, 1, 1), int(SemanticTokenTypes::Comment), 0);
+
+        /* single line comment    */
+        QTest::addRow("single-line-2") << fileItem
+                                       << Token(QQmlJS::SourceLocation(162, 28, 9, 1),
+                                                int(SemanticTokenTypes::Comment), 0);
+
+        // Multiline comments are split into multiple locations
+        QTest::addRow("multiline-first-line")
+                << fileItem
+                << Token(QQmlJS::SourceLocation(133, 2, 5, 1), int(SemanticTokenTypes::Comment), 0);
+        QTest::addRow("multiline-second-line") << fileItem
+                                               << Token(QQmlJS::SourceLocation(136, 21, 6, 1),
+                                                        int(SemanticTokenTypes::Comment), 0);
+        QTest::addRow("multiline-third-line")
+                << fileItem
+                << Token(QQmlJS::SourceLocation(158, 2, 7, 1), int(SemanticTokenTypes::Comment), 0);
+
+        // Comments Inside Js blocks
+        QTest::addRow("inside-js") << fileItem
+                                   << Token(QQmlJS::SourceLocation(232, 5, 13, 9),
+                                            int(SemanticTokenTypes::Comment), 0);
+    }
+}
+
+void tst_qmlls_highlighting::highlights()
+{
+    using namespace QQmlJS::Dom;
+    QFETCH(DomItem, fileItem);
+    QFETCH(Token, expectedHighlightedToken);
+
+    Highlights h;
+    HighlightingVisitor hv(h);
+
+    fileItem.visitTree(QQmlJS::Dom::Path(), hv, VisitOption::Default, emptyChildrenVisitor,
+                   emptyChildrenVisitor);
+
+    const auto highlights = h.highlights();
+    QVERIFY(highlights.contains(expectedHighlightedToken.offset));
+    QCOMPARE(highlights.value(expectedHighlightedToken.offset), expectedHighlightedToken);
+}
+
 QTEST_MAIN(tst_qmlls_highlighting)
diff --git a/tests/auto/qmlls/utils/tst_qmlls_highlighting.h b/tests/auto/qmlls/utils/tst_qmlls_highlighting.h
index 3348660265..7e6c6431ac 100644
--- a/tests/auto/qmlls/utils/tst_qmlls_highlighting.h
+++ b/tests/auto/qmlls/utils/tst_qmlls_highlighting.h
@@ -16,6 +16,11 @@ public:
 private slots:
     void encodeSemanticTokens_data();
     void encodeSemanticTokens();
+    void sourceLocationsFromMultiLineToken_data();
+    void sourceLocationsFromMultiLineToken();
+
+    void highlights_data();
+    void highlights();
 
 private:
     QString m_highlightingDataDir;