From 871832b8b67eb55208031f4b7edb6c02b824ad00 Mon Sep 17 00:00:00 2001 From: Katja Marttila Date: Thu, 26 Jan 2023 14:34:57 +0200 Subject: Optimize Updates.xml parsing Updates.xml file elements are read and based on that the installer calculates for example applicable updates. The files were parsed one by one which was not very optimal at least in Windows when reading a lot of files. Fixing the reading so that all files can be read and parsed using QtConcurrent. Also changing the class structure so that the private class is removed and private functions used instead. Another option would have been to create header file to the private class due to the Q_OBJECT definition in ParseXMLFilesTask. Task-number: QTIFW-2805 Change-Id: I40d1c75b87f4bf1f5a8fcd874edf84023891bdcc Reviewed-by: Arttu Tarkiainen --- src/libs/kdtools/updatefinder.cpp | 400 ++++++++++++++++++++------------------ src/libs/kdtools/updatefinder.h | 80 +++++++- src/libs/kdtools/updatesinfo.cpp | 5 + src/libs/kdtools/updatesinfo_p.h | 1 + 4 files changed, 289 insertions(+), 197 deletions(-) (limited to 'src/libs') diff --git a/src/libs/kdtools/updatefinder.cpp b/src/libs/kdtools/updatefinder.cpp index a45e9e3d0..120dcb952 100644 --- a/src/libs/kdtools/updatefinder.cpp +++ b/src/libs/kdtools/updatefinder.cpp @@ -1,7 +1,7 @@ /**************************************************************************** ** ** Copyright (C) 2013 Klaralvdalens Datakonsult AB (KDAB) -** Copyright (C) 2022 The Qt Company Ltd. +** Copyright (C) 2023 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the Qt Installer Framework. @@ -40,6 +40,7 @@ #include #include #include +#include using namespace KDUpdater; using namespace QInstaller; @@ -55,72 +56,107 @@ using namespace QInstaller; objects. */ -// -// Private -// -class UpdateFinder::Private + +static int computeProgressPercentage(int min, int max, int percent) { -public: - enum struct Resolution { - AddPackage, - KeepExisting, - RemoveExisting - }; - - explicit Private(UpdateFinder *qq) - : q(qq) - , cancel(false) - , downloadCompleteCount(0) - , m_downloadsToComplete(0) - {} - - ~Private() - { - clear(); - } + return min + qint64(max-min) * percent / 100; +} - struct Data { - Data() - : downloader(0) {} - explicit Data(const PackageSource &i, FileDownloader *d = 0) - : info(i), downloader(d) {} +static int computePercent(int done, int total) +{ + return total ? done * Q_INT64_C(100) / total : 0 ; +} - PackageSource info; - FileDownloader *downloader; - }; - UpdateFinder *q; - QHash updates; +/*! + Constructs an update finder. +*/ +UpdateFinder::UpdateFinder() + : Task(QLatin1String("UpdateFinder"), Stoppable) + , m_cancel(false) + , m_downloadCompleteCount(0) + , m_downloadsToComplete(0) + , m_updatesXmlTasks(0) + , m_updatesXmlTasksToComplete(0) +{ +} - // Temporary structure that notes down information about updates. - bool cancel; - int downloadCompleteCount; - int m_downloadsToComplete; - QHash m_updatesInfoList; +/*! + Destructor +*/ +UpdateFinder::~UpdateFinder() +{ + clear(); +} - void clear(); - void computeUpdates(); - void cancelComputeUpdates(); - bool downloadUpdateXMLFiles(); - bool computeApplicableUpdates(); +/*! + Returns a list of KDUpdater::Update objects. +*/ +QList UpdateFinder::updates() const +{ + return m_updates.values(); +} - QList applicableUpdates(UpdatesInfo *updatesInfo); - void createUpdateObjects(const PackageSource &source, const QList &updateInfoList); - Resolution checkPriorityAndVersion(const PackageSource &source, const QVariantHash &data) const; - void slotDownloadDone(); +/*! + Sets the information about installed local packages \a hub. +*/ +void UpdateFinder::setLocalPackageHub(std::weak_ptr hub) +{ + m_localPackageHub = std::move(hub); +} - QSet packageSources; - std::weak_ptr m_localPackageHub; -}; +/*! + Sets the package \a sources information when searching for applicable packages. +*/ +void UpdateFinder::setPackageSources(const QSet &sources) +{ + m_packageSources = sources; +} +/*! + \internal -static int computeProgressPercentage(int min, int max, int percent) + Implemented from KDUpdater::Task::doRun(). +*/ +void UpdateFinder::doRun() { - return min + qint64(max-min) * percent / 100; + computeUpdates(); } -static int computePercent(int done, int total) +/*! + \internal + + Implemented from KDUpdater::Task::doStop(). +*/ +bool UpdateFinder::doStop() { - return total ? done * Q_INT64_C(100) / total : 0 ; + cancelComputeUpdates(); + + // Wait until the cancel has actually happened, and then return. + // Thinking of using QMutex for this. Frank/Till any suggestions? + + return true; +} + +/*! + \internal + + Implemented from KDUpdater::Task::doStop(). +*/ +bool UpdateFinder::doPause() +{ + // Not a pausable task + return false; +} + +/*! + \internal + + Implemented from KDUpdater::Task::doStop(). +*/ +bool UpdateFinder::doResume() +{ + // Not a pausable task, hence it is not resumable as well + return false; } /*! @@ -128,20 +164,22 @@ static int computePercent(int done, int total) Releases all internal resources consumed while downloading and computing updates. */ -void UpdateFinder::Private::clear() +void UpdateFinder::clear() { - qDeleteAll(updates); - updates.clear(); + qDeleteAll(m_updates); + m_updates.clear(); const QList values = m_updatesInfoList.values(); foreach (const Data &data, values) delete data.downloader; - qDeleteAll(m_updatesInfoList.keys()); + qDeleteAll(m_updatesInfoList.keyBegin(), m_updatesInfoList.keyEnd()); m_updatesInfoList.clear(); - downloadCompleteCount = 0; + m_downloadCompleteCount = 0; m_downloadsToComplete = 0; + qDeleteAll(m_xmlFileTasks); + m_xmlFileTasks.clear(); } /*! @@ -164,51 +202,59 @@ void UpdateFinder::Private::clear() \note Each time this function is called, all the previously computed updates are discarded and its resources are freed. */ -void UpdateFinder::Private::computeUpdates() +void UpdateFinder::computeUpdates() { // Computing updates is done in two stages // 1. Downloading Update XML files from all the update sources - // 2. Matching updates with Package XML and figuring out available updates - + // 2. Parse attributes from Update XML documents to UpdateInfoList + // 3. Remove all updates with invalid content + // 4. Matching updates with Package XML and figuring out available updates clear(); - cancel = false; + m_cancel = false; // First do some quick sanity checks on the packages info std::shared_ptr packages = m_localPackageHub.lock(); if (!packages) { - q->reportError(tr("Cannot access the package information of this application.")); + reportError(tr("Cannot access the package information of this application.")); return; } if (!packages->isValid()) { - q->reportError(packages->errorString()); + reportError(packages->errorString()); return; } // Now do some quick sanity checks on the package sources. - if (packageSources.count() <= 0) { - q->reportError(tr("No package sources set for this application.")); + if (m_packageSources.count() <= 0) { + reportError(tr("No package sources set for this application.")); return; } // Now we can start... + if (!downloadUpdateXMLFiles() || m_cancel) { + clear(); + return; + } - // Step 1: 0 - 49 percent - if (!downloadUpdateXMLFiles() || cancel) { + if (!parseUpdateXMLFiles() || m_cancel) { clear(); return; } - // Step 2: 50 - 100 percent - if (!computeApplicableUpdates() || cancel) { + if (!removeInvalidObjects() || m_cancel) { + clear(); + return; + } + + if (!computeApplicableUpdates() || m_cancel) { clear(); return; } // All done - q->reportProgress(100, tr("%n update(s) found.", "", updates.count())); - q->reportDone(); + reportProgress(100, tr("%n update(s) found.", "", m_updates.count())); + reportDone(); } /*! @@ -218,9 +264,9 @@ void UpdateFinder::Private::computeUpdates() \sa computeUpdates() */ -void UpdateFinder::Private::cancelComputeUpdates() +void UpdateFinder::cancelComputeUpdates() { - cancel = true; + m_cancel = true; } /*! @@ -239,22 +285,22 @@ void UpdateFinder::Private::cancelComputeUpdates() The function gets into an event loop until all the downloads are complete. */ -bool UpdateFinder::Private::downloadUpdateXMLFiles() +bool UpdateFinder::downloadUpdateXMLFiles() { // create UpdatesInfo for each update source - foreach (const PackageSource &info, packageSources) { + foreach (const PackageSource &info, m_packageSources) { const QUrl url = QString::fromLatin1("%1/Updates.xml").arg(info.url.toString()); if (url.scheme() != QLatin1String("resource") && url.scheme() != QLatin1String("file")) { // create FileDownloader (except for local files and resources) - FileDownloader *downloader = FileDownloaderFactory::instance().create(url.scheme(), q); + FileDownloader *downloader = FileDownloaderFactory::instance().create(url.scheme(), this); if (!downloader) break; downloader->setUrl(url); downloader->setAutoRemoveDownloadedFile(true); - connect(downloader, SIGNAL(downloadCanceled()), q, SLOT(slotDownloadDone())); - connect(downloader, SIGNAL(downloadCompleted()), q, SLOT(slotDownloadDone())); - connect(downloader, SIGNAL(downloadAborted(QString)), q, SLOT(slotDownloadDone())); + connect(downloader, SIGNAL(downloadCanceled()), this, SLOT(slotDownloadDone())); + connect(downloader, SIGNAL(downloadCompleted()), this, SLOT(slotDownloadDone())); + connect(downloader, SIGNAL(downloadAborted(QString)), this, SLOT(slotDownloadDone())); m_updatesInfoList.insert(new UpdatesInfo, Data(info, downloader)); } else { UpdatesInfo *updatesInfo = new UpdatesInfo; @@ -264,7 +310,7 @@ bool UpdateFinder::Private::downloadUpdateXMLFiles() } // Trigger download of Updates.xml file - downloadCompleteCount = 0; + m_downloadCompleteCount = 0; m_downloadsToComplete = 0; foreach (const Data &data, m_updatesInfoList) { if (data.downloader) { @@ -274,39 +320,53 @@ bool UpdateFinder::Private::downloadUpdateXMLFiles() } // Wait until all downloaders have completed their downloads. - while (true) { - QCoreApplication::processEvents(); - if (cancel) - return false; - - if (downloadCompleteCount == m_downloadsToComplete) - break; - - q->reportProgress(computePercent(downloadCompleteCount, m_downloadsToComplete), - tr("Downloading Updates.xml from update sources.")); - } + return waitForJobToFinish(m_downloadCompleteCount, m_downloadsToComplete); +} +/*! + \internal +*/ +bool UpdateFinder::parseUpdateXMLFiles() +{ // Setup the update info objects with the files from download. - foreach (UpdatesInfo *updatesInfo, m_updatesInfoList.keys()) { + m_updatesXmlTasks = 0; + m_updatesXmlTasksToComplete = 0; + QList keys = m_updatesInfoList.keys(); + for (UpdatesInfo *updatesInfo : qAsConst(keys)) { const Data data = m_updatesInfoList.value(updatesInfo); if (data.downloader) { if (!data.downloader->isDownloaded()) { - q->reportError(tr("Cannot download package source %1 from \"%2\".").arg(data - .downloader->url().fileName(), data.info.url.toString())); + reportError(tr("Cannot download package source %1 from \"%2\".").arg(data. + downloader->url().fileName(), data.info.url.toString())); } else { updatesInfo->setFileName(data.downloader->downloadedFileName()); } } + if (!updatesInfo->fileName().isEmpty()) { + ParseXmlFilesTask *const task = new ParseXmlFilesTask(updatesInfo); + m_xmlFileTasks.append(task); + QFutureWatcher *watcher = new QFutureWatcher(); + m_updatesXmlTasksToComplete++; + connect(watcher, &QFutureWatcherBase::finished, this, &UpdateFinder::parseUpdatesXmlTaskFinished); + watcher->setFuture(QtConcurrent::run(&ParseXmlFilesTask::doTask, task)); + } } - // Remove all invalid update info objects. + // Wait until all updates.xml files are parsed + return waitForJobToFinish(m_updatesXmlTasks, m_updatesXmlTasksToComplete); +} +/*! + \internal +*/ +bool UpdateFinder::removeInvalidObjects() +{ QMutableHashIterator it(m_updatesInfoList); while (it.hasNext()) { UpdatesInfo *info = it.next().key(); if (info->isValid()) continue; - q->reportError(info->errorString()); + reportError(info->errorString()); delete info; it.remove(); } @@ -314,7 +374,7 @@ bool UpdateFinder::Private::downloadUpdateXMLFiles() if (m_updatesInfoList.isEmpty()) return false; - q->reportProgress(49, tr("Updates.xml file(s) downloaded from update sources.")); + reportProgress(49, tr("Updates.xml file(s) downloaded from update sources.")); return true; } @@ -326,36 +386,40 @@ bool UpdateFinder::Private::downloadUpdateXMLFiles() KDUpdater::PackagesInfo. Thereby figures out whether an update is applicable for this application or not. */ -bool UpdateFinder::Private::computeApplicableUpdates() +bool UpdateFinder::computeApplicableUpdates() { int i = 0; - foreach (UpdatesInfo *updatesInfo, m_updatesInfoList.keys()) { + QList keys = m_updatesInfoList.keys(); + for (UpdatesInfo *updatesInfo : qAsConst(keys)) { // Fetch updates applicable to this application. QList updates = applicableUpdates(updatesInfo); if (!updates.count()) continue; - if (cancel) + if (m_cancel) return false; const PackageSource updateSource = m_updatesInfoList.value(updatesInfo).info; // Create Update objects for updates that have a valid // UpdateFile createUpdateObjects(updateSource, updates); - if (cancel) + if (m_cancel) return false; // Report progress - q->reportProgress(computeProgressPercentage(51, 100, computePercent(i, + reportProgress(computeProgressPercentage(51, 100, computePercent(i, m_updatesInfoList.count())), tr("Computing applicable updates.")); ++i; } - q->reportProgress(99, tr("Application updates computed.")); + reportProgress(99, tr("Application updates computed.")); return true; } -QList UpdateFinder::Private::applicableUpdates(UpdatesInfo *updatesInfo) +/*! + \internal +*/ +QList UpdateFinder::applicableUpdates(UpdatesInfo *updatesInfo) { const QList dummy; if (!updatesInfo || updatesInfo->updateInfoCount() == 0) @@ -384,7 +448,10 @@ QList UpdateFinder::Private::applicableUpdates(UpdatesInfo *updatesI return updatesInfo->updatesInfo(); } -void UpdateFinder::Private::createUpdateObjects(const PackageSource &source, +/*! + \internal +*/ +void UpdateFinder::createUpdateObjects(const PackageSource &source, const QList &updateInfoList) { foreach (const UpdateInfo &info, updateInfoList) { @@ -394,23 +461,24 @@ void UpdateFinder::Private::createUpdateObjects(const PackageSource &source, const QString name = info.data.value(QLatin1String("Name")).toString(); if (value == Resolution::RemoveExisting) - delete updates.take(name); + delete m_updates.take(name); // Create and register the update - updates.insert(name, new Update(source, info)); + m_updates.insert(name, new Update(source, info)); } } -/* +/*! + \internal If a package of the same name exists, always use the one with the higher version. If the new package has the same version but a higher priority, use the new new package, otherwise keep the already existing package. */ -UpdateFinder::Private::Resolution UpdateFinder::Private::checkPriorityAndVersion( +UpdateFinder::Resolution UpdateFinder::checkPriorityAndVersion( const PackageSource &source, const QVariantHash &newPackage) const { const QString name = newPackage.value(QLatin1String("Name")).toString(); - if (Update *existingPackage = updates.value(name)) { + if (Update *existingPackage = m_updates.value(name)) { // Bingo, package was previously found elsewhere. const int match = compareVersion(newPackage.value(QLatin1String("Version")).toString(), @@ -442,108 +510,51 @@ UpdateFinder::Private::Resolution UpdateFinder::Private::checkPriorityAndVersion return Resolution::AddPackage; } -// -// UpdateFinder -// - -/*! - Constructs an update finder. -*/ -UpdateFinder::UpdateFinder() - : Task(QLatin1String("UpdateFinder"), Stoppable), - d(new Private(this)) -{ -} - -/*! - Destructor -*/ -UpdateFinder::~UpdateFinder() -{ - delete d; -} - -/*! - Returns a list of KDUpdater::Update objects. -*/ -QList UpdateFinder::updates() const -{ - return d->updates.values(); -} - -/*! - Sets the information about installed local packages \a hub. -*/ -void UpdateFinder::setLocalPackageHub(std::weak_ptr hub) -{ - d->m_localPackageHub = std::move(hub); -} - -/*! - Sets the package \a sources information when searching for applicable packages. -*/ -void UpdateFinder::setPackageSources(const QSet &sources) -{ - d->packageSources = sources; -} - -/*! - \internal - - Implemented from KDUpdater::Task::doRun(). -*/ -void UpdateFinder::doRun() -{ - d->computeUpdates(); -} - /*! \internal - - Implemented from KDUpdater::Task::doStop(). */ -bool UpdateFinder::doStop() +bool UpdateFinder::waitForJobToFinish(const int ¤tCount, const int &totalsCount) { - d->cancelComputeUpdates(); + while (true) { + QCoreApplication::processEvents(); + if (m_cancel) + return false; - // Wait until the cancel has actually happened, and then return. - // Thinking of using QMutex for this. Frank/Till any suggestions? + if (currentCount == totalsCount) + break; + reportProgress(computePercent(currentCount, totalsCount), + tr("Downloading Updates.xml from update sources.")); + } return true; } - /*! \internal - - Implemented from KDUpdater::Task::doStop(). */ -bool UpdateFinder::doPause() +void UpdateFinder::parseUpdatesXmlTaskFinished() { - // Not a pausable task - return false; -} + ++m_updatesXmlTasks; -/*! - \internal + int pc = computePercent(m_updatesXmlTasks, m_updatesXmlTasksToComplete); + pc = computeProgressPercentage(0, 45, pc); + reportProgress( pc, tr("Downloading Updates.xml from update sources.") ); - Implemented from KDUpdater::Task::doStop(). -*/ -bool UpdateFinder::doResume() -{ - // Not a pausable task, hence it is not resumable as well - return false; + QFutureWatcher *watcher = static_cast *>(sender()); + watcher->waitForFinished(); + watcher->deleteLater(); } + /*! \internal */ -void UpdateFinder::Private::slotDownloadDone() +void UpdateFinder::slotDownloadDone() { - ++downloadCompleteCount; + ++m_downloadCompleteCount; - int pc = computePercent(downloadCompleteCount, m_downloadsToComplete); + int pc = computePercent(m_downloadCompleteCount, m_downloadsToComplete); pc = computeProgressPercentage(0, 45, pc); - q->reportProgress( pc, tr("Downloading Updates.xml from update sources.") ); + reportProgress( pc, tr("Downloading Updates.xml from update sources.") ); } @@ -589,8 +600,9 @@ int KDUpdater::compareVersion(const QString &v1, const QString &v2) return 0; // Split version components across ".", "-" or "_" - QStringList v1_comps = v1.split(QRegularExpression(QLatin1String( "\\.|-|_"))); - QStringList v2_comps = v2.split(QRegularExpression(QLatin1String( "\\.|-|_"))); + static const QRegularExpression regex(QLatin1String( "\\.|-|_")); + QStringList v1_comps = v1.split(regex); + QStringList v2_comps = v2.split(regex); // Check each component of the version int index = 0; diff --git a/src/libs/kdtools/updatefinder.h b/src/libs/kdtools/updatefinder.h index 5a3f50f62..626a700fd 100644 --- a/src/libs/kdtools/updatefinder.h +++ b/src/libs/kdtools/updatefinder.h @@ -1,7 +1,7 @@ /**************************************************************************** ** ** Copyright (C) 2013 Klaralvdalens Datakonsult AB (KDAB) -** Copyright (C) 2022 The Qt Company Ltd. +** Copyright (C) 2023 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the Qt Installer Framework. @@ -32,19 +32,68 @@ #include "task.h" #include "packagesource.h" +#include "filedownloader.h" +#include "updatesinfo_p.h" +#include "abstracttask.h" #include +using namespace QInstaller; namespace KDUpdater { class LocalPackageHub; class Update; +class ParseXmlFilesTask : public AbstractTask +{ + Q_OBJECT + Q_DISABLE_COPY(ParseXmlFilesTask) + +public: + ParseXmlFilesTask(UpdatesInfo *info) + : m_info(info) + {} + + void doTask(QFutureInterface &fi) override + { + fi.reportStarted(); + fi.setExpectedResultCount(1); + + if (fi.isCanceled()) { + fi.reportFinished(); + return; // ignore already canceled + } + m_info->parseFile(); + + fi.reportFinished(); + } + +private: + UpdatesInfo *const m_info; +}; + + class KDTOOLS_EXPORT UpdateFinder : public Task { Q_OBJECT class Private; + struct Data { + Data() + : downloader(0) {} + explicit Data(const QInstaller::PackageSource &i, KDUpdater::FileDownloader *d = 0) + : info(i), downloader(d) {} + + QInstaller::PackageSource info; + KDUpdater::FileDownloader *downloader; + }; + + enum struct Resolution { + AddPackage, + KeepExisting, + RemoveExisting + }; + public: UpdateFinder(); ~UpdateFinder(); @@ -59,10 +108,35 @@ private: bool doStop() override; bool doPause() override; bool doResume() override; + void clear(); + void computeUpdates(); + void cancelComputeUpdates(); + bool downloadUpdateXMLFiles(); + bool parseUpdateXMLFiles(); + bool removeInvalidObjects(); + bool computeApplicableUpdates(); + + QList applicableUpdates(UpdatesInfo *updatesInfo); + void createUpdateObjects(const PackageSource &source, const QList &updateInfoList); + Resolution checkPriorityAndVersion(const QInstaller::PackageSource &source, const QVariantHash &data) const; + bool waitForJobToFinish(const int ¤tCount, const int &totalsCount); + +private slots: + void parseUpdatesXmlTaskFinished(); + void slotDownloadDone(); private: - Private *d; - Q_PRIVATE_SLOT(d, void slotDownloadDone()) + QSet m_packageSources; + std::weak_ptr m_localPackageHub; + QHash m_updates; + + bool m_cancel; + int m_downloadCompleteCount; + int m_downloadsToComplete; + QHash m_updatesInfoList; + int m_updatesXmlTasks; + int m_updatesXmlTasksToComplete; + QList m_xmlFileTasks; }; } // namespace KDUpdater diff --git a/src/libs/kdtools/updatesinfo.cpp b/src/libs/kdtools/updatesinfo.cpp index 8d70f54eb..707daf11f 100644 --- a/src/libs/kdtools/updatesinfo.cpp +++ b/src/libs/kdtools/updatesinfo.cpp @@ -1,6 +1,7 @@ /**************************************************************************** ** ** Copyright (C) 2013 Klaralvdalens Datakonsult AB (KDAB) +** Copyright (C) 2023 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the Qt Installer Framework. @@ -266,6 +267,10 @@ void UpdatesInfo::setFileName(const QString &updateXmlFile) d->updateInfoList.clear(); d->updateXmlFile = updateXmlFile; +} + +void UpdatesInfo::parseFile() +{ d->parseFile(d->updateXmlFile); } diff --git a/src/libs/kdtools/updatesinfo_p.h b/src/libs/kdtools/updatesinfo_p.h index 8b9928ed9..bd9885327 100644 --- a/src/libs/kdtools/updatesinfo_p.h +++ b/src/libs/kdtools/updatesinfo_p.h @@ -69,6 +69,7 @@ public: QString fileName() const; void setFileName(const QString &updateXmlFile); + void parseFile(); QString applicationName() const; QString applicationVersion() const; -- cgit v1.2.3