Add support for skipping fetching already cached Updates.xml files

This speeds up the metadata evaluation with the Qt online installer,
which uses a dynamic list of temporary repositories that can be
provided with an expected checksum information. The installer or
maintenance tool can skip downloading the Updates.xml files again for
already cached metadata items, in case the checksum of the file in the
repository has not changed since the last metadata fetch.

Task-number: QTIFW-2873
Change-Id: I61384d8bcbd29e01fda3019fb00d7360b0f81435
Reviewed-by: Katja Marttila <katja.marttila@qt.io>

5 files changed, 100 insertions, 14 deletions

diff --git a/src/libs/installer/constants.h b/src/libs/installer/constants.h
index e887fcba1..1ac1d8940 100644
--- a/src/libs/installer/constants.h
+++ b/src/libs/installer/constants.h
@@ -41,6 +41,7 @@ static const QLatin1String scScript("script");
 static const QLatin1String scAllUsersStartMenuProgramsPath("AllUsersStartMenuProgramsPath");
 static const QLatin1String scUserStartMenuProgramsPath("UserStartMenuProgramsPath");
 static const QLatin1String scUILanguage("UILanguage");
+static const QLatin1String scUpdatesXML("Updates.xml");
 
 static const QLatin1String scName("Name");
 static const QLatin1String scVersion("Version");
diff --git a/src/libs/installer/metadatajob.cpp b/src/libs/installer/metadatajob.cpp
index 3bb47ca8b..0254f32d0 100644
--- a/src/libs/installer/metadatajob.cpp
+++ b/src/libs/installer/metadatajob.cpp
@@ -225,6 +225,8 @@ void MetadataJob::doStart()
     setError(Job::NoError);
     setErrorString(QString());
     m_metadataResult.clear();
+    setProgressTotalAmount(100);
+
     if (!m_core) {
         emitFinishedWithError(Job::Canceled, tr("Missing package manager core engine."));
         return; // We can't do anything here without core, so avoid tons of !m_core checks.
@@ -239,28 +241,56 @@ void MetadataJob::doStart()
         emit infoMessage(this, tr("Fetching latest update information..."));
         const bool onlineInstaller = m_core->isInstaller() && !m_core->isOfflineOnly();
         if (onlineInstaller || m_core->isMaintainer()) {
+            static const QString updateFilePath(QLatin1Char('/') + scUpdatesXML + QLatin1Char('?'));
+            static const QString randomQueryString = QString::number(QRandomGenerator::global()->generate());
+
             QList<FileTaskItem> items;
             QSet<Repository> repositories = getRepositories();
+            quint64 cachedCount = 0;
             foreach (const Repository &repo, repositories) {
+                // For not blocking the UI
+                qApp->processEvents();
+
                 if (repo.isEnabled() &&
                         productKeyCheck->isValidRepository(repo)) {
                     QAuthenticator authenticator;
                     authenticator.setUser(repo.username());
                     authenticator.setPassword(repo.password());
 
-                    if (!repo.isCompressed()) {
-                        QString url = repo.url().toString() + QLatin1String("/Updates.xml?");
-                        if (!m_core->value(scUrlQueryString).isEmpty())
-                            url += m_core->value(scUrlQueryString) + QLatin1Char('&');
-
-                        // also append a random string to avoid proxy caches
-                        FileTaskItem item(url.append(QString::number(QRandomGenerator::global()->generate())));
-                        item.insert(TaskRole::UserRole, QVariant::fromValue(repo));
-                        item.insert(TaskRole::Authenticator, QVariant::fromValue(authenticator));
-                        items.append(item);
+                    if (repo.isCompressed())
+                        continue;
+
+                    QString url;
+                    url = repo.url().toString() + updateFilePath;
+                    if (!m_core->value(scUrlQueryString).isEmpty())
+                        url += m_core->value(scUrlQueryString) + QLatin1Char('&');
+                    // also append a random string to avoid proxy caches
+                    url.append(randomQueryString);
+
+                    // Check if we can skip downloading already cached repositories
+                    const Status foundStatus = findCachedUpdatesFile(repo, url);
+                    if (foundStatus == XmlDownloadSuccess) {
+                        // Found existing Updates.xml
+                        ++cachedCount;
+                        continue;
+                    } else if (foundStatus == XmlDownloadRetry) {
+                         // Repositories changed, restart with the new repositories
+                        QMetaObject::invokeMethod(this, "doStart", Qt::QueuedConnection);
+                        return;
                     }
+
+                    FileTaskItem item(url);
+                    item.insert(TaskRole::UserRole, QVariant::fromValue(repo));
+                    item.insert(TaskRole::Authenticator, QVariant::fromValue(authenticator));
+                    items.append(item);
                 }
             }
+            const quint64 totalCount = repositories.count();
+            if (cachedCount > 0) {
+                qCDebug(lcInstallerInstallLog).nospace() << "Loaded from cache "
+                    << cachedCount << "/" << totalCount << ". Downloading remaining "
+                    << items.count() << "/" << totalCount <<".";
+            }
             if (items.count() > 0) {
                 startXMLTask(items);
             } else {
@@ -305,7 +335,6 @@ void MetadataJob::startXMLTask(const QList<FileTaskItem> &items)
 {
     DownloadFileTask *const xmlTask = new DownloadFileTask(items);
     xmlTask->setProxyFactory(m_core->proxyFactory());
-    setProgressTotalAmount(100);
     connect(&m_xmlTask, &QFutureWatcher<FileTaskResult>::progressValueChanged, this,
         &MetadataJob::progressChanged);
     m_xmlTask.setFuture(QtConcurrent::run(&DownloadFileTask::doTask, xmlTask));
@@ -885,6 +914,31 @@ MetadataJob::Status MetadataJob::refreshCacheItem(const FileTaskResult &result,
     return XmlDownloadSuccess;
 }
 
+MetadataJob::Status MetadataJob::findCachedUpdatesFile(const Repository &repository, const QString &fileUrl)
+{
+    if (repository.xmlChecksum().isEmpty())
+        return XmlDownloadFailure;
+
+    Metadata *metadata = m_metaFromCache.itemByChecksum(repository.xmlChecksum());
+    if (!metadata)
+        return XmlDownloadFailure;
+
+    const QString targetPath = metadata->path() + QLatin1Char('/') + scUpdatesXML;
+
+    FileTaskItem cachedMetaTaskItem(fileUrl, targetPath);
+    cachedMetaTaskItem.insert(TaskRole::UserRole, QVariant::fromValue(repository));
+    const FileTaskResult cachedMetaTaskResult(targetPath, repository.xmlChecksum(), cachedMetaTaskItem, false);
+
+    bool isCached = false;
+    const Status status = refreshCacheItem(cachedMetaTaskResult, repository.xmlChecksum(), &isCached);
+    if (isCached)
+        return XmlDownloadSuccess;
+    else if (status == XmlDownloadRetry)
+        return XmlDownloadRetry;
+    else
+        return XmlDownloadFailure;
+}
+
 MetadataJob::Status MetadataJob::parseRepositoryUpdates(const QDomElement &root,
     const FileTaskResult &result, Metadata *metadata)
 {
diff --git a/src/libs/installer/metadatajob.h b/src/libs/installer/metadatajob.h
index d2207ea38..4bc14c683 100644
--- a/src/libs/installer/metadatajob.h
+++ b/src/libs/installer/metadatajob.h
@@ -98,6 +98,7 @@ private:
     void resetCompressedFetch();
     Status parseUpdatesXml(const QList<FileTaskResult> &results);
     Status refreshCacheItem(const FileTaskResult &result, const QByteArray &checksum, bool *refreshed);
+    Status findCachedUpdatesFile(const Repository &repository, const QString &fileUrl);
     Status parseRepositoryUpdates(const QDomElement &root, const FileTaskResult &result, Metadata *metadata);
     QSet<Repository> getRepositories();
     void addFileTaskItem(const QString &source, const QString &target, Metadata *metadata,
diff --git a/src/libs/installer/repository.cpp b/src/libs/installer/repository.cpp
index e8df4e433..4ef8f9f25 100644
--- a/src/libs/installer/repository.cpp
+++ b/src/libs/installer/repository.cpp
@@ -64,6 +64,7 @@ Repository::Repository(const Repository &other)
     , m_displayname(other.m_displayname)
     , m_compressed(other.m_compressed)
     , m_categoryname(other.m_categoryname)
+    , m_xmlChecksum(other.m_xmlChecksum)
 {
 }
 
@@ -221,6 +222,28 @@ void Repository::setCategoryName(const QString &categoryname)
 }
 
 /*!
+    Returns the expected checksum of the repository, which is the checksum
+    calculated from the \c Updates.xml document at the root of the repository.
+
+    This value is used as a hint when looking for already fetched repositories
+    from the local cache. If the installer has cached a repository with a matching
+    checksum, it can skip downloading the \c Updates.xml file for that repository again.
+*/
+QByteArray Repository::xmlChecksum() const
+{
+    return m_xmlChecksum;
+}
+
+/*!
+    Sets the expected checksum of the repository to \c checksum. The checksum
+    is calculated from the \c Updates.xml document at the root of the repository.
+*/
+void Repository::setXmlChecksum(const QByteArray &checksum)
+{
+    m_xmlChecksum = checksum;
+}
+
+/*!
     Returns true if repository is compressed
 */
 bool Repository::isCompressed() const
@@ -235,7 +258,8 @@ bool Repository::isCompressed() const
 bool Repository::operator==(const Repository &other) const
 {
     return m_url == other.m_url && m_default == other.m_default && m_enabled == other.m_enabled
-        && m_username == other.m_username && m_password == other.m_password && m_displayname == other.m_displayname;
+        && m_username == other.m_username && m_password == other.m_password
+        && m_displayname == other.m_displayname && m_xmlChecksum == other.m_xmlChecksum;
 }
 
 /*!
@@ -263,6 +287,7 @@ const Repository &Repository::operator=(const Repository &other)
     m_displayname = other.m_displayname;
     m_compressed = other.m_compressed;
     m_categoryname = other.m_categoryname;
+    m_xmlChecksum = other.m_xmlChecksum;
 
     return *this;
 }
@@ -282,7 +307,7 @@ QDataStream &operator>>(QDataStream &istream, Repository &repository)
 {
     QByteArray url, username, password, displayname, compressed;
     istream >> url >> repository.m_default >> repository.m_enabled >> username >> password
-            >> displayname >> repository.m_categoryname;
+            >> displayname >> repository.m_categoryname >> repository.m_xmlChecksum;
     repository.setUrl(QUrl::fromEncoded(QByteArray::fromBase64(url)));
     repository.setUsername(QString::fromUtf8(QByteArray::fromBase64(username)));
     repository.setPassword(QString::fromUtf8(QByteArray::fromBase64(password)));
@@ -297,7 +322,8 @@ QDataStream &operator<<(QDataStream &ostream, const Repository &repository)
 {
     return ostream << repository.m_url.toEncoded().toBase64() << repository.m_default << repository.m_enabled
         << repository.m_username.toUtf8().toBase64() << repository.m_password.toUtf8().toBase64()
-        << repository.m_displayname.toUtf8().toBase64() << repository.m_categoryname.toUtf8().toBase64();
+        << repository.m_displayname.toUtf8().toBase64() << repository.m_categoryname.toUtf8().toBase64()
+        << repository.m_xmlChecksum.toBase64();
 }
 
 }
diff --git a/src/libs/installer/repository.h b/src/libs/installer/repository.h
index 3f28e4d99..1edb83449 100644
--- a/src/libs/installer/repository.h
+++ b/src/libs/installer/repository.h
@@ -67,6 +67,9 @@ public:
     QString categoryname() const;
     void setCategoryName(const QString &categoryname);
 
+    QByteArray xmlChecksum() const;
+    void setXmlChecksum(const QByteArray &checksum);
+
     bool isCompressed() const;
     bool operator==(const Repository &other) const;
     bool operator!=(const Repository &other) const;
@@ -86,6 +89,7 @@ private:
     QString m_displayname;
     QString m_categoryname;
     bool m_compressed;
+    QByteArray m_xmlChecksum;
 };
 
 inline uint qHash(const Repository &repository)