diff options
Diffstat (limited to 'tools/qmlls/qqmlcodemodel.cpp')
| -rw-r--r-- | tools/qmlls/qqmlcodemodel.cpp | 610 |
1 files changed, 610 insertions, 0 deletions
diff --git a/tools/qmlls/qqmlcodemodel.cpp b/tools/qmlls/qqmlcodemodel.cpp new file mode 100644 index 0000000000..77e0298180 --- /dev/null +++ b/tools/qmlls/qqmlcodemodel.cpp @@ -0,0 +1,610 @@ +/**************************************************************************** +** +** Copyright (C) 2021 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the qmllanguageserver tool of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include "qqmllanguageserver.h" +#include "qqmlcodemodel.h" +#include <QtCore/qfileinfo.h> +#include <QtCore/qdir.h> +#include <QtCore/qthreadpool.h> +#include <QtQmlDom/private/qqmldomtop_p.h> +#include "textdocument.h" + +#include <memory> + +QT_BEGIN_NAMESPACE + +namespace QmlLsp { + +Q_LOGGING_CATEGORY(codeModelLog, "qt.languageserver.codemodel") + +using namespace QQmlJS::Dom; + +/*! +\internal +\class QQmlCodeModel + +The code model offers a view of the current state of the current files, and traks open files. +All methods are threadsafe, and generally return immutable or threadsafe objects that can be +worked on from any thread (unless otherwise noted). +The idea is the let all other operations be as lock free as possible, concentrating all tricky +synchronization here. + +\section2 Global views +\list +\li currentEnv() offers a view that contains the latest version of all the loaded files +\li validEnv() is just like current env but stores only the valid (meaning correctly parsed, + not necessarily without errors) version of a file, it is normally a better choice to load the + dependencies/symbol information from +\endlist + +\section2 OpenFiles +\list +\li snapshotByUri() returns an OpenDocumentSnapshot of an open document. Form it you can get the + document, its latest valid version, scope,... and connected to a specific version of the document + and immutable. The signal updatedSnapshot() is called every time a snapshot changes (also for + every partial change: document change, validDocument change, scope change). +\li openDocumentByUri() is a lower level and more intrusive access to OpenDocument objects. These + contains the current snapshot, and shared pointer to a Utils::TextDocument. This is *always* the + current version of the document, and has line by line support. + Working on it is more delicate and intrusive, because you have to explicitly acquire its mutex() + before *any* read or write/modification to it. + It has a version nuber which is supposed to always change and increase. + It is mainly used for highlighting/indenting, and s immediately updated when the user edits a + document. Its use should be avoided if possible, preferring the snapshots. +\endlist + +\section2 Parallelism/Theading +Most operations are not parallel and usially take place in the main thread (but are still thread +safe). +There are two main task that are executed in parallel: Indexing, and OpenDocumentUpdate. +Indexing is meant to keep the global view up to date. +OpenDocumentUpdate keeps the snapshots of the open documents up to date. + +There is always a tension between bein responsive, using all threads available, and avoid to hog too +many resources. One can choose different parallelization strategies, we went with a flexiable +approach. +We have (private) functions that execute part of the work: indexSome() and openUpdateSome(). These +do all locking needed, get some work, do it without locks, and at the end update the state of the +code model. If there is more work, then they return true. Thus while (xxxSome()); works until there +is no work left. + +addDirectoriesToIndex(), the internal addDirectory() and addOpenToUpdate() add more work to do. + +indexNeedsUpdate() and openNeedUpdate(), check if there is work to do, and if yes ensure that a +worker thread (or more) that work on it exist. + +An initial implementation allowed one to register a callback to be called when a given open document +had some chosen parts of the snapshot up to date. But I did not need anythign more that the +updatedSnapshot() signal, so that has been removed, but something like that might become useful in +the future. +*/ + +QQmlCodeModel::QQmlCodeModel(QObject *parent) + : QObject { parent }, + m_currentEnv(std::make_shared<DomEnvironment>(QStringList(), + DomEnvironment::Option::SingleThreaded)), + m_validEnv(std::make_shared<DomEnvironment>(QStringList(), + DomEnvironment::Option::SingleThreaded)) +{ +} + +OpenDocumentSnapshot QQmlCodeModel::snapshotByUri(const QByteArray &uri) +{ + return openDocumentByUri(uri).snapshot; +} + +int QQmlCodeModel::indexEvalProgress() const +{ + // should be called with acquired mutex + const int dirCost = 10; + int costToDo = 1; + for (const ToIndex &el : qAsConst(m_toIndex)) + costToDo += dirCost * el.leftDepth; + costToDo += m_indexInProgressCost; + return m_indexDoneCost * 100 / (costToDo + m_indexDoneCost); +} + +void QQmlCodeModel::indexStart() +{ + Q_ASSERT(!m_mutex.tryLock()); // should be called while locked + qCDebug(codeModelLog) << "indexStart"; +} + +void QQmlCodeModel::indexEnd() +{ + Q_ASSERT(!m_mutex.tryLock()); // should be called while locked + qCDebug(codeModelLog) << "indexEnd"; + m_lastIndexProgress = 0; + m_nIndexInProgress = 0; + m_nUpdateInProgress = 0; + m_toIndex.clear(); + m_indexInProgressCost = 0; + m_indexDoneCost = 0; +} + +void QQmlCodeModel::indexSendProgress(int progress) +{ + if (progress <= m_lastIndexProgress) + return; + m_lastIndexProgress = progress; + // to do: send progress +} + +bool QQmlCodeModel::indexCancelled() +{ + return false; +} + +void QQmlCodeModel::indexDirectory(const QString &path, int depthLeft) +{ + if (indexCancelled()) + return; + QDir dir(path); + if (depthLeft > 1) { + const QStringList dirs = + dir.entryList(QDir::Dirs | QDir::NoDotAndDotDot | QDir::NoSymLinks); + for (const QString &child : dirs) + addDirectory(dir.filePath(child), --depthLeft); + } + const QStringList qmljs = dir.entryList(QStringList({ "*.qml", "*.js", "*.mjs" }), QDir::Files); + int progress = 0; + { + QMutexLocker l(&m_mutex); + m_indexInProgressCost += qmljs.size(); + progress = indexEvalProgress(); + } + indexSendProgress(progress); + if (qmljs.isEmpty()) + return; + DomItem newCurrent = m_currentEnv.makeCopy(DomItem::CopyOption::EnvConnected).item(); + int iFile = 0; + for (const QString &file : qmljs) { + if (indexCancelled()) + return; + QString fPath = dir.filePath(file); + QFileInfo fInfo(fPath); + QString cPath = fInfo.canonicalFilePath(); + if (!cPath.isEmpty()) { + bool isNew = false; + newCurrent.loadFile(cPath, fPath, + [&isNew](Path, DomItem &oldValue, DomItem &newValue) { + if (oldValue != newValue) + isNew = true; + }, + {}); + newCurrent.loadPendingDependencies(); + if (isNew) { + newCurrent.commitToBase(); + m_currentEnv.makeCopy(DomItem::CopyOption::EnvConnected).item(); + } + } + ++iFile; + { + QMutexLocker l(&m_mutex); + ++m_indexDoneCost; + --m_indexInProgressCost; + progress = indexEvalProgress(); + } + indexSendProgress(progress); + } +} + +void QQmlCodeModel::addDirectoriesToIndex(const QStringList &paths, QLanguageServer *server) +{ + Q_UNUSED(server); + // to do create progress, &scan in a separate instance + const int maxDepth = 5; + for (const auto &path : paths) + addDirectory(path, maxDepth); + indexNeedsUpdate(); +} + +void QQmlCodeModel::addDirectory(const QString &path, int depthLeft) +{ + if (depthLeft < 1) + return; + { + QMutexLocker l(&m_mutex); + auto it = m_toIndex.begin(); + auto end = m_toIndex.end(); + while (it != end) { + if (it->path.startsWith(path)) { + if (it->path.size() == path.size()) + return; + if (it->path.at(path.size()) == u'/') { + it = m_toIndex.erase(it); + continue; + } + } else if (path.startsWith(it->path) && path.at(it->path.size()) == u'/') + return; + } + m_toIndex.append({ path, depthLeft }); + } +} + +void QQmlCodeModel::removeDirectory(const QString &path) +{ + { + QMutexLocker l(&m_mutex); + auto toRemove = [path](const QString &p) { + return p.startsWith(path) && (p.size() == path.size() || p.at(path.size()) == u'/'); + }; + auto it = m_toIndex.begin(); + auto end = m_toIndex.end(); + while (it != end) { + if (toRemove(it->path)) + it = m_toIndex.erase(it); + else + ++it; + } + } + if (auto validEnvPtr = m_validEnv.ownerAs<DomEnvironment>()) + validEnvPtr->removePath(path); + if (auto currentEnvPtr = m_currentEnv.ownerAs<DomEnvironment>()) + currentEnvPtr->removePath(path); +} + +QString QQmlCodeModel::uri2Path(const QByteArray &uri, UriLookup options) +{ + QString res; + { + QMutexLocker l(&m_mutex); + res = m_uri2path.value(uri); + } + if (!res.isEmpty() && options == UriLookup::Caching) + return res; + QUrl url(QString::fromUtf8(uri)); + QFileInfo f(url.toLocalFile()); + QString cPath = f.canonicalFilePath(); + if (cPath.isEmpty()) + cPath = f.filePath(); + { + QMutexLocker l(&m_mutex); + if (!res.isEmpty() && res != cPath) + m_path2uri.remove(res); + m_uri2path.insert(uri, cPath); + m_path2uri.insert(cPath, uri); + } + return cPath; +} + +void QQmlCodeModel::newOpenFile(const QByteArray &uri, int version, const QString &docText) +{ + { + QMutexLocker l(&m_mutex); + auto &openDoc = m_openDocuments[uri]; + if (!openDoc.textDocument) + openDoc.textDocument = std::make_shared<Utils::TextDocument>(); + QMutexLocker l2(openDoc.textDocument->mutex()); + openDoc.textDocument->setVersion(version); + openDoc.textDocument->setPlainText(docText); + } + addOpenToUpdate(uri); + openNeedUpdate(); +} + +OpenDocument QQmlCodeModel::openDocumentByUri(const QByteArray &uri) +{ + QMutexLocker l(&m_mutex); + return m_openDocuments.value(uri); +} + +void QQmlCodeModel::indexNeedsUpdate() +{ + const int maxIndexThreads = 1; + { + QMutexLocker l(&m_mutex); + if (m_toIndex.isEmpty() || m_nIndexInProgress >= maxIndexThreads) + return; + if (++m_nIndexInProgress == 1) + indexStart(); + } + QThreadPool::globalInstance()->start([this]() { + while (indexSome()) { } + }); +} + +bool QQmlCodeModel::indexSome() +{ + qCDebug(codeModelLog) << "indexSome"; + ToIndex toIndex; + { + QMutexLocker l(&m_mutex); + if (m_toIndex.isEmpty()) { + if (--m_nIndexInProgress == 0) + indexEnd(); + return false; + } + toIndex = m_toIndex.last(); + m_toIndex.removeLast(); + } + bool hasMore = false; + auto guard = qScopeGuard([this, &hasMore]() { + QMutexLocker l(&m_mutex); + if (m_toIndex.isEmpty()) { + if (--m_nIndexInProgress == 0) + indexEnd(); + hasMore = false; + } else { + hasMore = true; + } + }); + indexDirectory(toIndex.path, toIndex.leftDepth); + return hasMore; +} + +void QQmlCodeModel::openNeedUpdate() +{ + qCDebug(codeModelLog) << "openNeedUpdate"; + const int maxIndexThreads = 1; + { + QMutexLocker l(&m_mutex); + if (m_openDocumentsToUpdate.isEmpty() || m_nUpdateInProgress >= maxIndexThreads) + return; + if (++m_nUpdateInProgress == 1) + openUpdateStart(); + } + QThreadPool::globalInstance()->start([this]() { + while (openUpdateSome()) { } + }); +} + +bool QQmlCodeModel::openUpdateSome() +{ + qCDebug(codeModelLog) << "openUpdateSome start"; + QByteArray toUpdate; + { + QMutexLocker l(&m_mutex); + if (m_openDocumentsToUpdate.isEmpty()) { + if (--m_nUpdateInProgress == 0) + openUpdateEnd(); + return false; + } + auto it = m_openDocumentsToUpdate.find(m_lastOpenDocumentUpdated); + auto end = m_openDocumentsToUpdate.end(); + if (it == end) + it = m_openDocumentsToUpdate.begin(); + else if (++it == end) + it = m_openDocumentsToUpdate.begin(); + toUpdate = *it; + m_openDocumentsToUpdate.erase(it); + } + bool hasMore = false; + { + auto guard = qScopeGuard([this, &hasMore]() { + QMutexLocker l(&m_mutex); + if (m_openDocumentsToUpdate.isEmpty()) { + if (--m_nUpdateInProgress == 0) + openUpdateEnd(); + hasMore = false; + } else { + hasMore = true; + } + }); + openUpdate(toUpdate); + } + return hasMore; +} + +void QQmlCodeModel::openUpdateStart() +{ + qCDebug(codeModelLog) << "openUpdateStart"; +} + +void QQmlCodeModel::openUpdateEnd() +{ + qCDebug(codeModelLog) << "openUpdateEnd"; +} + +DomItem QQmlCodeModel::validDocForUpdate(DomItem &item) +{ + if (item.field(Fields::isValid).value().toBool(false)) { + if (auto envPtr = m_validEnv.ownerAs<DomEnvironment>()) { + switch (item.fileObject().internalKind()) { + case DomType::QmlFile: + envPtr->addQmlFile(item.fileObject().ownerAs<QmlFile>()); + break; + case DomType::JsFile: + envPtr->addJsFile(item.fileObject().ownerAs<JsFile>()); + break; + default: + qCWarning(lspServerLog) + << "Unexpected file type " << item.fileObject().internalKindStr(); + return DomItem(); + } + return m_validEnv.path(item.canonicalPath()); + } + } + return DomItem(); +} + +void QQmlCodeModel::newDocForOpenFile(const QByteArray &uri, int version, const QString &docText) +{ + qCDebug(codeModelLog) << "updating doc" << uri << "to version" << version << "(" + << docText.length() << "chars)"; + DomItem newCurrent = m_currentEnv.makeCopy(DomItem::CopyOption::EnvConnected).item(); + QString fPath = uri2Path(uri, UriLookup::ForceLookup); + Path p; + newCurrent.loadFile( + fPath, fPath, docText, QDateTime::currentDateTimeUtc(), + [&p](Path, DomItem &, DomItem &newValue) { p = newValue.fileObject().canonicalPath(); }, + {}); + newCurrent.loadPendingDependencies(); + if (p) { + newCurrent.commitToBase(); + DomItem item = m_currentEnv.path(p); + DomItem vDoc = validDocForUpdate(item); + { + QMutexLocker l(&m_mutex); + OpenDocument &doc = m_openDocuments[uri]; + if (!doc.textDocument) { + qCWarning(lspServerLog) + << "ignoring update to closed document" << QString::fromUtf8(uri); + return; + } else { + QMutexLocker l(doc.textDocument->mutex()); + if (doc.textDocument->version() && *doc.textDocument->version() > version) { + qCWarning(lspServerLog) + << "docUpdate: version" << version << "of document" + << QString::fromUtf8(uri) << "is not the latest anymore"; + return; + } + } + if (!doc.snapshot.docVersion || *doc.snapshot.docVersion < version) { + doc.snapshot.docVersion = version; + doc.snapshot.doc = item; + } else { + qCWarning(lspServerLog) << "skippig update of current doc to obsolete version" + << version << "of document" << QString::fromUtf8(uri); + } + if (vDoc) { + if (!doc.snapshot.validDocVersion || *doc.snapshot.validDocVersion < version) { + doc.snapshot.validDocVersion = version; + doc.snapshot.validDoc = vDoc; + } else { + qCWarning(lspServerLog) << "skippig update of valid doc to obsolete version" + << version << "of document" << QString::fromUtf8(uri); + } + } else { + qCWarning(lspServerLog) + << "avoid update of validDoc to " << version << "of document" + << QString::fromUtf8(uri) << "as it is invalid"; + } + } + } + if (codeModelLog().isDebugEnabled()) { + qCDebug(codeModelLog) << "finished update doc of " << uri << "to version" << version; + snapshotByUri(uri).dump(qDebug() << "postSnapshot", + OpenDocumentSnapshot::DumpOption::AllCode); + } + // we should update the scope in the future thus call addOpen(uri) + emit updatedSnapshot(uri); +} + +void QQmlCodeModel::closeOpenFile(const QByteArray &uri) +{ + QMutexLocker l(&m_mutex); + m_openDocuments.remove(uri); +} + +void QQmlCodeModel::openUpdate(const QByteArray &uri) +{ + bool updateDoc = false; + bool updateScope = false; + std::optional<int> rNow = 0; + QString docText; + DomItem validDoc; + std::shared_ptr<Utils::TextDocument> document; + { + QMutexLocker l(&m_mutex); + OpenDocument &doc = m_openDocuments[uri]; + document = doc.textDocument; + if (!document) + return; + { + QMutexLocker l2(document->mutex()); + rNow = document->version(); + } + if (rNow && (!doc.snapshot.docVersion || *doc.snapshot.docVersion != *rNow)) + updateDoc = true; + else if (doc.snapshot.validDocVersion + && (!doc.snapshot.scopeVersion + || *doc.snapshot.scopeVersion != *doc.snapshot.validDocVersion)) + updateScope = true; + else + return; + if (updateDoc) { + QMutexLocker l2(doc.textDocument->mutex()); + rNow = doc.textDocument->version(); + docText = doc.textDocument->toPlainText(); + } else { + validDoc = doc.snapshot.validDoc; + rNow = doc.snapshot.validDocVersion; + } + } + if (updateDoc) { + newDocForOpenFile(uri, *rNow, docText); + } + if (updateScope) { + // to do + } +} + +void QQmlCodeModel::addOpenToUpdate(const QByteArray &uri) +{ + QMutexLocker l(&m_mutex); + m_openDocumentsToUpdate.insert(uri); +} + +QDebug OpenDocumentSnapshot::dump(QDebug dbg, DumpOptions options) +{ + dbg.noquote().nospace() << "{"; + dbg << " uri:" << QString::fromUtf8(uri) << "\n"; + dbg << " docVersion:" << (docVersion ? QString::number(*docVersion) : u"*none*"_qs) << "\n"; + if (options & DumpOption::LatestCode) { + dbg << " doc: ------------\n" + << doc.field(Fields::code).value().toString() << "\n==========\n"; + } else { + dbg << u" doc:" + << (doc ? u"%1chars"_qs.arg(doc.field(Fields::code).value().toString().length()) + : u"*none*"_qs) + << "\n"; + } + dbg << " validDocVersion:" + << (validDocVersion ? QString::number(*validDocVersion) : u"*none*"_qs) << "\n"; + if (options & DumpOption::ValidCode) { + dbg << " validDoc: ------------\n" + << validDoc.field(Fields::code).value().toString() << "\n==========\n"; + } else { + dbg << u" validDoc:" + << (validDoc ? u"%1chars"_qs.arg( + validDoc.field(Fields::code).value().toString().length()) + : u"*none*"_qs) + << "\n"; + } + dbg << " scopeVersion:" << (scopeVersion ? QString::number(*scopeVersion) : u"*none*"_qs) + << "\n"; + dbg << " scopeDependenciesLoadTime:" << scopeDependenciesLoadTime << "\n"; + dbg << " scopeDependenciesChanged" << scopeDependenciesChanged << "\n"; + dbg << "}"; + return dbg; +} + +} // namespace QmlLsp + +QT_END_NAMESPACE |