diff options
Diffstat (limited to 'src/libs/installer/metadatajob.cpp')
| -rw-r--r-- | src/libs/installer/metadatajob.cpp | 677 |
1 files changed, 489 insertions, 188 deletions
diff --git a/src/libs/installer/metadatajob.cpp b/src/libs/installer/metadatajob.cpp index 68a242ebc..1bed304c6 100644 --- a/src/libs/installer/metadatajob.cpp +++ b/src/libs/installer/metadatajob.cpp @@ -1,6 +1,6 @@ /************************************************************************** ** -** Copyright (C) 2020 The Qt Company Ltd. +** Copyright (C) 2024 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the Qt Installer Framework. @@ -38,31 +38,18 @@ #include "globals.h" #include <QTemporaryDir> +#include <QtConcurrent> #include <QtMath> #include <QRandomGenerator> - -const QStringList metaElements = {QLatin1String("Script"), QLatin1String("Licenses"), QLatin1String("UserInterfaces"), QLatin1String("Translations")}; +#include <QApplication> namespace QInstaller { /*! - \inmodule QtInstallerFramework - \class QInstaller::Metadata - \internal -*/ - -/*! - \inmodule QtInstallerFramework - \class QInstaller::ArchiveMetadata - \internal -*/ - -/*! \enum QInstaller::DownloadType \value All \value CompressedPackage - \value UpdatesXML */ /*! @@ -97,6 +84,7 @@ MetadataJob::MetadataJob(QObject *parent) , m_downloadType(DownloadType::All) , m_downloadableChunkSize(1000) , m_taskNumber(0) + , m_defaultRepositoriesFetched(false) { QByteArray downloadableChunkSize = qgetenv("IFW_METADATA_SIZE"); if (!downloadableChunkSize.isEmpty()) { @@ -109,12 +97,19 @@ MetadataJob::MetadataJob(QObject *parent) connect(&m_xmlTask, &QFutureWatcherBase::finished, this, &MetadataJob::xmlTaskFinished); connect(&m_metadataTask, &QFutureWatcherBase::finished, this, &MetadataJob::metadataTaskFinished); connect(&m_metadataTask, &QFutureWatcherBase::progressValueChanged, this, &MetadataJob::progressChanged); + connect(&m_updateCacheTask, &QFutureWatcherBase::finished, this, &MetadataJob::updateCacheTaskFinished); } MetadataJob::~MetadataJob() { resetCompressedFetch(); reset(); + + if (!m_core) + return; + + if (m_metaFromCache.isValid() && !m_core->settings().persistentLocalCache()) + m_metaFromCache.clear(); } /* @@ -123,26 +118,101 @@ MetadataJob::~MetadataJob() * repositories which might not be currently selected. */ -QList<Metadata> MetadataJob::metadata() const +QList<Metadata *> MetadataJob::metadata() const { - QList<Metadata> metadata = m_metaFromDefaultRepositories.values(); - foreach (RepositoryCategory repositoryCategory, m_core->settings().repositoryCategories()) { - if (m_core->isUpdater() || (repositoryCategory.isEnabled() && m_fetchedArchive.contains(repositoryCategory.displayname()))) { - QList<ArchiveMetadata> archiveMetaList = m_fetchedArchive.values(repositoryCategory.displayname()); - foreach (ArchiveMetadata archiveMeta, archiveMetaList) { - metadata.append(archiveMeta.metaData); - } + const QSet<RepositoryCategory> categories = m_core->settings().repositoryCategories(); + QHash<RepositoryCategory, QSet<Repository>> repositoryHash; + // Create hash of categorized repositories to avoid constructing + // excess temp objects when filtering below. + for (const RepositoryCategory &category : categories) + repositoryHash.insert(category, category.repositories()); + + QList<Metadata *> metadata = m_metaFromCache.items(); + // Filter cache items not associated with current repositories and categories + QtConcurrent::blockingFilter(metadata, [&](const Metadata *item) { + if (!item->isActive()) + return false; + + // No need to check if the repository is enabled here. Changing the network + // settings resets the cache and we don't fetch the disabled repositories, + // so the cached items stay inactive. + + if (item->isAvailableFromDefaultRepository()) + return true; + + QHash<RepositoryCategory, QSet<Repository>>::const_iterator it; + for (it = repositoryHash.constBegin(); it != repositoryHash.constEnd(); ++it) { + if (!it.key().isEnabled()) + continue; // Let's try the next one + + if (it->contains(item->repository())) + return true; } - } + return false; + }); + return metadata; } -Repository MetadataJob::repositoryForDirectory(const QString &directory) const +/* + Returns a repository object from the cache item matching \a directory. If the + \a directory does not belong to the cache, an empty repository is returned. +*/ +Repository MetadataJob::repositoryForCacheDirectory(const QString &directory) const { - if (m_metaFromDefaultRepositories.contains(directory)) - return m_metaFromDefaultRepositories.value(directory).repository; - else - return m_metaFromArchive.value(directory).repository; + QDir dir(directory); + if (!QDir::fromNativeSeparators(dir.path()) + .startsWith(QDir::fromNativeSeparators(m_metaFromCache.path()))) { + return Repository(); + } + const QString dirName = dir.dirName(); + Metadata *cachedMeta = m_metaFromCache.itemByChecksum(dirName.toUtf8()); + if (cachedMeta) + return cachedMeta->repository(); + + return Repository(); +} + +bool MetadataJob::resetCache(bool init) +{ + // Need the path from current settings + if (!m_core) { + qCWarning(lcInstallerInstallLog) << "Cannot reset metadata cache: " + "missing package manager core engine."; + return false; + } + + if (m_metaFromCache.isValid() && !m_core->settings().persistentLocalCache()) + m_metaFromCache.clear(); + + m_metaFromCache.setPath(m_core->settings().localCachePath()); + + if (!init) + return true; + + const bool success = m_metaFromCache.initialize(); + if (success) { + qCDebug(QInstaller::lcInstallerInstallLog) << "Using metadata cache from" + << m_metaFromCache.path(); + qCDebug(QInstaller::lcInstallerInstallLog) << "Found" + << m_metaFromCache.items().count() << "cached items."; + } + return success; +} + +bool MetadataJob::clearCache() +{ + if (m_metaFromCache.clear()) + return true; + + setError(JobError::CacheError); + setErrorString(m_metaFromCache.errorString()); + return false; +} + +bool MetadataJob::isValidCache() const +{ + return m_metaFromCache.isValid(); } // -- private slots @@ -151,39 +221,91 @@ void MetadataJob::doStart() { setError(Job::NoError); setErrorString(QString()); + m_metadataResult.clear(); + setProgressTotalAmount(100); + if (!m_core) { emitFinishedWithError(Job::Canceled, tr("Missing package manager core engine.")); return; // We can't do anything here without core, so avoid tons of !m_core checks. } + if (!m_metaFromCache.isValid() && !resetCache(true)) { + emitFinishedWithError(JobError::CacheError, m_metaFromCache.errorString()); + return; + } + const ProductKeyCheck *const productKeyCheck = ProductKeyCheck::instance(); - if (m_downloadType == DownloadType::All || m_downloadType == DownloadType::UpdatesXML) { - emit infoMessage(this, tr("Preparing meta information download...")); + if (m_downloadType != DownloadType::CompressedPackage) { + emit infoMessage(this, tr("Fetching latest update information...")); const bool onlineInstaller = m_core->isInstaller() && !m_core->isOfflineOnly(); - if (onlineInstaller || m_core->isMaintainer()) { - QList<FileTaskItem> items; - QSet<Repository> repositories = getRepositories(); + const QSet<Repository> repositories = getRepositories(); + + if (onlineInstaller || m_core->isMaintainer() + || (m_core->settings().allowRepositoriesForOfflineInstaller() && !repositories.isEmpty())) { + static const QString updateFilePath(QLatin1Char('/') + scUpdatesXML + QLatin1Char('?')); + static const QString randomQueryString = QString::number(QRandomGenerator::global()->generate()); + + quint64 cachedCount = 0; + setProgressTotalAmount(0); // Show only busy indicator during this loop as we have no progress to measure foreach (const Repository &repo, repositories) { + // For not blocking the UI + qApp->processEvents(); + if (repo.isEnabled() && productKeyCheck->isValidRepository(repo)) { QAuthenticator authenticator; authenticator.setUser(repo.username()); authenticator.setPassword(repo.password()); - if (!repo.isCompressed()) { - QString url = repo.url().toString() + QLatin1String("/Updates.xml?"); - if (!m_core->value(scUrlQueryString).isEmpty()) - url += m_core->value(scUrlQueryString) + QLatin1Char('&'); + if (repo.isCompressed()) + continue; + + QString url; + url = repo.url().toString() + updateFilePath; + if (!m_core->value(scUrlQueryString).isEmpty()) + url += m_core->value(scUrlQueryString) + QLatin1Char('&'); + // also append a random string to avoid proxy caches + url.append(randomQueryString); + + // Check if we can skip downloading already cached repositories + const Status foundStatus = findCachedUpdatesFile(repo, url); + if (foundStatus == XmlDownloadSuccess) { + // Found existing Updates.xml + ++cachedCount; + continue; + } else if (foundStatus == XmlDownloadRetry) { + // Repositories changed, restart with the new repositories + QMetaObject::invokeMethod(this, "doStart", Qt::QueuedConnection); + return; + } - // also append a random string to avoid proxy caches - FileTaskItem item(url.append(QString::number(QRandomGenerator::global()->generate()))); - item.insert(TaskRole::UserRole, QVariant::fromValue(repo)); - item.insert(TaskRole::Authenticator, QVariant::fromValue(authenticator)); - items.append(item); + QTemporaryDir tmp(QDir::tempPath() + QLatin1String("/remoterepo-XXXXXX")); + if (!tmp.isValid()) { + qCWarning(QInstaller::lcInstallerInstallLog) << "Cannot create unique temporary directory."; + continue; } + tmp.setAutoRemove(false); + m_tempDirDeleter.add(tmp.path()); + FileTaskItem item(url, tmp.path() + QLatin1String("/Updates.xml")); + item.insert(TaskRole::UserRole, QVariant::fromValue(repo)); + item.insert(TaskRole::Authenticator, QVariant::fromValue(authenticator)); + m_updatesXmlItems.append(item); } } - if (items.count() > 0) { - startXMLTask(items); + setProgressTotalAmount(100); + const quint64 totalCount = repositories.count(); + if (cachedCount > 0) { + qCDebug(lcInstallerInstallLog).nospace() << "Loaded from cache " + << cachedCount << "/" << totalCount << ". Downloading remaining " + << m_updatesXmlItems.count() << "/" << totalCount <<"."; + } else { + qCDebug(lcInstallerInstallLog).nospace() <<"Downloading " << m_updatesXmlItems.count() + << " items to cache."; + } + if (m_updatesXmlItems.count() > 0) { + double taskCount = m_updatesXmlItems.length()/static_cast<double>(m_downloadableChunkSize); + m_totalTaskCount = qCeil(taskCount); + m_taskNumber = 0; + startXMLTask(); } else { emitFinished(); } @@ -222,18 +344,28 @@ void MetadataJob::doStart() } } -void MetadataJob::startXMLTask(const QList<FileTaskItem> &items) +bool MetadataJob::startXMLTask() { - DownloadFileTask *const xmlTask = new DownloadFileTask(items); - xmlTask->setProxyFactory(m_core->proxyFactory()); - connect(&m_xmlTask, &QFutureWatcher<FileTaskResult>::progressValueChanged, this, - &MetadataJob::progressChanged); - m_xmlTask.setFuture(QtConcurrent::run(&DownloadFileTask::doTask, xmlTask)); + int chunkSize = qMin(m_updatesXmlItems.length(), m_downloadableChunkSize); + QList<FileTaskItem> tempPackages = m_updatesXmlItems.mid(0, chunkSize); + m_updatesXmlItems = m_updatesXmlItems.mid(chunkSize, m_updatesXmlItems.length()); + if (tempPackages.length() > 0) { + DownloadFileTask *const xmlTask = new DownloadFileTask(tempPackages); + xmlTask->setProxyFactory(m_core->proxyFactory()); + connect(&m_xmlTask, &QFutureWatcher<FileTaskResult>::progressValueChanged, this, + &MetadataJob::progressChanged); + m_xmlTask.setFuture(QtConcurrent::run(&DownloadFileTask::doTask, xmlTask)); + + setInfoMessage(tr("Retrieving information from remote repositories...")); + return true; + } + return false; } void MetadataJob::doCancel() { reset(); + resetCache(); emitFinishedWithError(Job::Canceled, tr("Metadata download canceled.")); } @@ -257,6 +389,27 @@ void MetadataJob::startUnzipRepositoryTask(const Repository &repo) watcher->setFuture(QtConcurrent::run(&UnzipArchiveTask::doTask, task)); } +void MetadataJob::startUpdateCacheTask() +{ + const int toRegisterCount = m_fetchedMetadata.count(); + if (toRegisterCount > 0) + emit infoMessage(this, tr("Updating local cache with %n new items...", + nullptr, toRegisterCount)); + + UpdateCacheTask *task = new UpdateCacheTask(m_metaFromCache, m_fetchedMetadata); + m_updateCacheTask.setFuture(QtConcurrent::run(&UpdateCacheTask::doTask, task)); +} + +/* + Resets the repository information from all cache items, which + makes them inactive until associated with new repositories. +*/ +void MetadataJob::resetCacheRepositories() +{ + for (auto *metaToReset : m_metaFromCache.items()) + metaToReset->setRepository(Repository()); +} + void MetadataJob::unzipRepositoryTaskFinished() { QFutureWatcher<void> *watcher = static_cast<QFutureWatcher<void> *>(sender()); @@ -283,9 +436,17 @@ void MetadataJob::unzipRepositoryTaskFinished() error = testJob.error(); errorString = testJob.errorString(); if (error == Job::NoError) { - FileTaskItem item(url); + QTemporaryDir tmp(QDir::tempPath() + QLatin1String("/remoterepo-XXXXXX")); + if (!tmp.isValid()) { + qCWarning(QInstaller::lcInstallerInstallLog) << "Cannot create unique temporary directory."; + continue; + } + tmp.setAutoRemove(false); + m_tempDirDeleter.add(tmp.path()); + FileTaskItem item(url, tmp.path() + QLatin1String("/Updates.xml")); + item.insert(TaskRole::UserRole, QVariant::fromValue(repo)); - m_unzipRepositoryitems.append(item); + m_updatesXmlItems.append(item); } else { //Repository is not valid, remove it Settings &s = m_core->settings(); @@ -305,8 +466,8 @@ void MetadataJob::unzipRepositoryTaskFinished() //One can specify many zipped repository items at once. As the repositories are //unzipped one by one, we collect here all items before parsing xml files from those. - if (m_unzipRepositoryitems.count() > 0 && m_unzipRepositoryTasks.isEmpty()) { - startXMLTask(m_unzipRepositoryitems); + if (m_updatesXmlItems.count() > 0 && m_unzipRepositoryTasks.isEmpty()) { + startXMLTask(); } else { if (error != Job::NoError) { emitFinishedWithError(QInstaller::DownloadError, errorString); @@ -330,19 +491,34 @@ void MetadataJob::xmlTaskFinished() Status status = XmlDownloadFailure; try { m_xmlTask.waitForFinished(); - status = parseUpdatesXml(m_xmlTask.future().results()); + m_updatesXmlResult.append(m_xmlTask.future().results()); + if (!startXMLTask()) { + status = parseUpdatesXml(m_updatesXmlResult); + m_updatesXmlResult.clear(); + } else { + return; + } } catch (const AuthenticationRequiredException &e) { if (e.type() == AuthenticationRequiredException::Type::Proxy) { - const QNetworkProxy proxy = e.proxy(); - ProxyCredentialsDialog proxyCredentials(proxy); qCWarning(QInstaller::lcInstallerInstallLog) << e.message(); - - if (proxyCredentials.exec() == QDialog::Accepted) { + QString username; + QString password; + const QNetworkProxy proxy = e.proxy(); + if (m_core->isCommandLineInstance()) { + qCDebug(QInstaller::lcInstallerInstallLog).noquote() << QString::fromLatin1("The proxy %1:%2 requires a username and password").arg(proxy.hostName(), proxy.port()); + askForCredentials(&username, &password, QLatin1String("Username: "), QLatin1String("Password: ")); + } else { + ProxyCredentialsDialog proxyCredentials(proxy); + if (proxyCredentials.exec() == QDialog::Accepted) { + username = proxyCredentials.userName(); + password = proxyCredentials.password(); + } + } + if (!username.isEmpty()) { qCDebug(QInstaller::lcInstallerInstallLog) << "Retrying with new credentials ..."; PackageManagerProxyFactory *factory = m_core->proxyFactory(); - factory->setProxyCredentials(proxy, proxyCredentials.userName(), - proxyCredentials.password()); + factory->setProxyCredentials(proxy, username, password); m_core->setProxyFactory(factory); status = XmlDownloadRetry; } else { @@ -351,13 +527,25 @@ void MetadataJob::xmlTaskFinished() } } else if (e.type() == AuthenticationRequiredException::Type::Server) { qCWarning(QInstaller::lcInstallerInstallLog) << e.message(); - ServerAuthenticationDialog dlg(e.message(), e.taskItem()); - if (dlg.exec() == QDialog::Accepted) { + QString username; + QString password; + if (m_core->isCommandLineInstance()) { + qCDebug(QInstaller::lcInstallerInstallLog) << "Server Requires Authentication"; + qCDebug(QInstaller::lcInstallerInstallLog) << "You need to supply a username and password to access this site."; + askForCredentials(&username, &password, QLatin1String("Username: "), QLatin1String("Password: ")); + } else { + ServerAuthenticationDialog dlg(e.message(), e.taskItem()); + if (dlg.exec() == QDialog::Accepted) { + username = dlg.user(); + password = dlg.password(); + } + } + if (!username.isEmpty()) { Repository original = e.taskItem().value(TaskRole::UserRole) .value<Repository>(); Repository replacement = original; - replacement.setUsername(dlg.user()); - replacement.setPassword(dlg.password()); + replacement.setUsername(username); + replacement.setPassword(password); Settings &s = m_core->settings(); QSet<Repository> temporaries = s.temporaryRepositories(); @@ -366,7 +554,7 @@ void MetadataJob::xmlTaskFinished() temporaries.insert(replacement); s.addTemporaryRepositories(temporaries, true); } else { - QHash<QString, QPair<Repository, Repository> > update; + QMultiHash<QString, QPair<Repository, Repository> > update; update.insert(QLatin1String("replace"), qMakePair(original, replacement)); if (s.updateRepositoryCategories(update) == Settings::UpdatesApplied) @@ -407,13 +595,13 @@ void MetadataJob::xmlTaskFinished() return; if (status == XmlDownloadSuccess) { - if (m_downloadType != DownloadType::UpdatesXML) { - if (!fetchMetaDataPackages()) - emitFinished(); - } else { - emitFinished(); + if (!fetchMetaDataPackages()) { + // No new metadata packages to fetch, still need to update the cache + // for refreshed repositories. + startUpdateCacheTask(); } } else if (status == XmlDownloadRetry) { + reset(); QMetaObject::invokeMethod(this, "doStart", Qt::QueuedConnection); } else { reset(); @@ -441,10 +629,8 @@ void MetadataJob::unzipTaskFinished() m_unzipTasks.remove(watcher); delete watcher; - if (m_unzipTasks.isEmpty()) { - setProcessedAmount(100); - emitFinished(); - } + if (m_unzipTasks.isEmpty()) + startUpdateCacheTask(); } void MetadataJob::progressChanged(int progress) @@ -476,17 +662,29 @@ void MetadataJob::metadataTaskFinished() } else { throw QInstaller::TaskException(mismatchMessage); } + QFileInfo fi(result.target()); + QString targetPath = fi.absolutePath(); + if (m_fetchedMetadata.contains(targetPath)) { + delete m_fetchedMetadata.value(targetPath); + m_fetchedMetadata.remove(targetPath); + } + continue; } UnzipArchiveTask *task = new UnzipArchiveTask(result.target(), item.value(TaskRole::UserRole).toString()); + task->setRemoveArchive(true); + task->setStoreChecksums(true); QFutureWatcher<void> *watcher = new QFutureWatcher<void>(); m_unzipTasks.insert(watcher, qobject_cast<QObject*> (task)); connect(watcher, &QFutureWatcherBase::finished, this, &MetadataJob::unzipTaskFinished); watcher->setFuture(QtConcurrent::run(&UnzipArchiveTask::doTask, task)); } + if (m_unzipTasks.isEmpty()) + startUpdateCacheTask(); + } else { - emitFinished(); + startUpdateCacheTask(); } } } catch (const TaskException &e) { @@ -501,6 +699,25 @@ void MetadataJob::metadataTaskFinished() } } +void MetadataJob::updateCacheTaskFinished() +{ + try { + m_updateCacheTask.waitForFinished(); + } catch (const CacheTaskException &e) { + emitFinishedWithError(QInstaller::CacheError, e.message()); + } catch (const QUnhandledException &e) { + emitFinishedWithError(QInstaller::CacheError, QLatin1String(e.what())); + } catch (...) { + emitFinishedWithError(QInstaller::CacheError, tr("Unknown exception during updating cache.")); + } + + if (error() != Job::NoError) + return; + + setProcessedAmount(100); + emitFinished(); +} + // -- private @@ -511,18 +728,11 @@ bool MetadataJob::fetchMetaDataPackages() QList<FileTaskItem> tempPackages = m_packages.mid(0, chunkSize); m_packages = m_packages.mid(chunkSize, m_packages.length()); if (tempPackages.length() > 0) { - m_taskNumber++; setProcessedAmount(0); DownloadFileTask *const metadataTask = new DownloadFileTask(tempPackages); metadataTask->setProxyFactory(m_core->proxyFactory()); m_metadataTask.setFuture(QtConcurrent::run(&DownloadFileTask::doTask, metadataTask)); - setProgressTotalAmount(100); - QString metaInformation; - if (m_totalTaskCount > 1) - metaInformation = tr("Retrieving meta information from remote repository... %1/%2 ").arg(m_taskNumber).arg(m_totalTaskCount); - else - metaInformation = tr("Retrieving meta information from remote repository... "); - emit infoMessage(this, metaInformation); + setInfoMessage(tr("Retrieving meta information from remote repository...")); return true; } return false; @@ -531,9 +741,12 @@ bool MetadataJob::fetchMetaDataPackages() void MetadataJob::reset() { m_packages.clear(); - m_metaFromDefaultRepositories.clear(); - m_metaFromArchive.clear(); - m_fetchedArchive.clear(); + m_updatesXmlItems.clear(); + m_defaultRepositoriesFetched = false; + m_fetchedCategorizedRepositories.clear(); + + qDeleteAll(m_fetchedMetadata); + m_fetchedMetadata.clear(); setError(Job::NoError); setErrorString(QString()); @@ -547,6 +760,7 @@ void MetadataJob::reset() } catch (...) {} m_tempDirDeleter.releaseAndDeleteAll(); m_metadataResult.clear(); + m_updatesXmlResult.clear(); m_taskNumber = 0; } @@ -554,7 +768,6 @@ void MetadataJob::resetCompressedFetch() { setError(Job::NoError); setErrorString(QString()); - m_unzipRepositoryitems.clear(); try { foreach (QFutureWatcher<void> *const watcher, m_unzipTasks.keys()) { @@ -586,43 +799,54 @@ MetadataJob::Status MetadataJob::parseUpdatesXml(const QList<FileTaskResult> &re if (result.target().isEmpty()) { continue; } - Metadata metadata; - QTemporaryDir tmp(QDir::tempPath() + QLatin1String("/remoterepo-XXXXXX")); - if (!tmp.isValid()) { - qCWarning(QInstaller::lcInstallerInstallLog) << "Cannot create unique temporary directory."; - return XmlDownloadFailure; - } - - tmp.setAutoRemove(false); - metadata.directory = tmp.path(); - m_tempDirDeleter.add(metadata.directory); + QFileInfo fileInfo(result.target()); + std::unique_ptr<Metadata> metadata(new Metadata(fileInfo.absolutePath())); QFile file(result.target()); - if (!file.rename(metadata.directory + QLatin1String("/Updates.xml"))) { - qCWarning(QInstaller::lcInstallerInstallLog) << "Cannot rename target to Updates.xml:" - << file.errorString(); - return XmlDownloadFailure; - } - if (!file.open(QIODevice::ReadOnly)) { qCWarning(QInstaller::lcInstallerInstallLog) << "Cannot open Updates.xml for reading:" << file.errorString(); return XmlDownloadFailure; } + const FileTaskItem item = result.value(TaskRole::TaskItem).value<FileTaskItem>(); + const Repository repository = item.value(TaskRole::UserRole).value<Repository>(); + + QCryptographicHash hash(QCryptographicHash::Sha1); + hash.addData(&file); + const QByteArray updatesChecksum = hash.result().toHex(); + + if (!repository.xmlChecksum().isEmpty() && updatesChecksum != repository.xmlChecksum()) { + qCWarning(lcDeveloperBuild).noquote().nospace() << "The checksum for Updates.xml " + "file downloaded from repository:\n" << repository.url().toString() << "\ndoes not " + "match the expected value:\n\tActual SHA1: " << updatesChecksum << "\n\tExpected SHA1: " + << repository.xmlChecksum() << Qt::endl; + } + + bool refreshed; + // Check if we have cached the metadata for this repository already + Status status = refreshCacheItem(result, updatesChecksum, &refreshed); + if (status != XmlDownloadSuccess) + return status; + + if (refreshed) // Found existing metadata + continue; + + metadata->setChecksum(updatesChecksum); + + file.seek(0); QString error; QDomDocument doc; if (!doc.setContent(&file, &error)) { qCWarning(QInstaller::lcInstallerInstallLog).nospace() << "Cannot fetch a valid version of Updates.xml from repository " - << metadata.repository.displayname() << ": " << error; + << metadata->repository().displayname() << ": " << error; //If there are other repositories, try to use those continue; } file.close(); - const FileTaskItem item = result.value(TaskRole::TaskItem).value<FileTaskItem>(); - metadata.repository = item.value(TaskRole::UserRole).value<Repository>(); - const bool online = !(metadata.repository.url().scheme()).isEmpty(); + metadata->setRepository(repository); + const bool online = !(metadata->repository().url().scheme()).isEmpty(); bool testCheckSum = true; const QDomElement root = doc.documentElement(); @@ -634,14 +858,14 @@ MetadataJob::Status MetadataJob::parseUpdatesXml(const QList<FileTaskResult> &re // all metadata inside one repository to a single 7z file. Fetch that // instead of component specific meta 7z files. const QDomNode sha1 = root.firstChildElement(scSHA1); - QDomElement metadataNameElement = root.firstChildElement(QLatin1String("MetadataName")); + QDomElement metadataNameElement = root.firstChildElement(scMetadataName); QDomNodeList children = root.childNodes(); if (!sha1.isNull() && !metadataNameElement.isNull()) { - const QString repoUrl = metadata.repository.url().toString(); - const QString metadataName = metadataNameElement.toElement().text(); - addFileTaskItem(QString::fromLatin1("%1/%2").arg(repoUrl, metadataName), - metadata.directory + QString::fromLatin1("/%1").arg(metadataName), - metadata, sha1.toElement().text(), QString()); + const QString repoUrl = metadata->repository().url().toString(); + const QString metadataName = metadataNameElement.toElement().text(); + addFileTaskItem(QString::fromLatin1("%1/%2").arg(repoUrl, metadataName), + metadata->path() + QString::fromLatin1("/%1").arg(metadataName), + metadata.get(), sha1.toElement().text(), QString()); } else { bool metaFound = false; for (int i = 0; i < children.count(); ++i) { @@ -653,16 +877,13 @@ MetadataJob::Status MetadataJob::parseUpdatesXml(const QList<FileTaskResult> &re online, testCheckSum); // If meta element (script, licenses, etc.) is not found, no need to fetch metadata. - // The offline-generator instance is an exception to this - if the Updates.xml contains - // checksum element for the meta-archive, we will fetch it, so that the temporary - // location contents match the remote repository. - if (metaFound || (m_core->isOfflineGenerator() && !packageHash.isEmpty())) { - const QString repoUrl = metadata.repository.url().toString(); + if (metaFound) { + const QString repoUrl = metadata->repository().url().toString(); addFileTaskItem(QString::fromLatin1("%1/%2/%3meta.7z").arg(repoUrl, packageName, packageVersion), - metadata.directory + QString::fromLatin1("/%1-%2-meta.7z").arg(packageName, packageVersion), - metadata, packageHash, packageName); + metadata->path() + QString::fromLatin1("/%1-%2-meta.7z").arg(packageName, packageVersion), + metadata.get(), packageHash, packageName); } else { - QString fileName = metadata.directory + QLatin1Char('/') + packageName; + QString fileName = metadata->path() + QLatin1Char('/') + packageName; QDir directory(fileName); if (!directory.exists()) { directory.mkdir(fileName); @@ -672,40 +893,24 @@ MetadataJob::Status MetadataJob::parseUpdatesXml(const QList<FileTaskResult> &re } } - if (metadata.repository.categoryname().isEmpty()) { - m_metaFromDefaultRepositories.insert(metadata.directory, metadata); - } else { - //Hash metadata to help checking if meta for repository is already fetched - ArchiveMetadata archiveMetadata; - archiveMetadata.metaData = metadata; - m_fetchedArchive.insertMulti(metadata.repository.categoryname(), archiveMetadata); - - //Check if other categories have the same url (contains same metadata) - //so we can speed up other category fetches - foreach (RepositoryCategory category, m_core->settings().repositoryCategories()) { - if (category.displayname() != metadata.repository.categoryname()) { - foreach (Repository repository, category.repositories()) { - if (repository.url() == metadata.repository.url()) { - m_fetchedArchive.insertMulti(category.displayname(), archiveMetadata); - } - } - } - } - // Hash for faster lookups - m_metaFromArchive.insert(metadata.directory, metadata); - } + // Remember the fetched metadata + Metadata *const metadataPtr = metadata.get(); + const QString categoryName = metadata->repository().categoryname(); + if (categoryName.isEmpty()) + metadata->setAvailableFromDefaultRepository(true); + else + m_fetchedCategorizedRepositories.insert(metadataPtr->repository()); // For faster lookups + const QString metadataPath = metadata->path(); + m_fetchedMetadata.insert(metadataPath, metadata.release()); // search for additional repositories that we might need to check - const QDomNode repositoryUpdate = root.firstChildElement(QLatin1String("RepositoryUpdate")); - if (!repositoryUpdate.isNull()) { - QHash<QString, QPair<Repository, Repository> > repositoryUpdates = - searchAdditionalRepositories(repositoryUpdate, result, metadata); - if (!repositoryUpdates.isEmpty()) { - MetadataJob::Status status = setAdditionalRepositories(repositoryUpdates, result, metadata); - if (status == XmlDownloadRetry) - return status; - } + status = parseRepositoryUpdates(root, result, metadataPtr); + if (status == XmlDownloadRetry) { + // The repository update may have removed or replaced current repositories, + // clear repository information from cached items and refresh on next fetch run. + resetCacheRepositories(); + return status; } } double taskCount = m_packages.length()/static_cast<double>(m_downloadableChunkSize); @@ -715,46 +920,128 @@ MetadataJob::Status MetadataJob::parseUpdatesXml(const QList<FileTaskResult> &re return XmlDownloadSuccess; } +MetadataJob::Status MetadataJob::refreshCacheItem(const FileTaskResult &result, + const QByteArray &checksum, bool *refreshed) +{ + Q_ASSERT(refreshed); + *refreshed = false; + + Metadata *cachedMetadata = m_metaFromCache.itemByChecksum(checksum); + if (!cachedMetadata) + return XmlDownloadSuccess; + + const FileTaskItem item = result.value(TaskRole::TaskItem).value<FileTaskItem>(); + const Repository repository = item.value(TaskRole::UserRole).value<Repository>(); + + if (cachedMetadata->isValid() && !repository.isCompressed()) { + // Refresh repository information to cache. Same repository may appear in multiple + // categories and the metadata may be available from default repositories simultaneously. + cachedMetadata->setRepository(repository); + if (!repository.categoryname().isEmpty()) + m_fetchedCategorizedRepositories.insert(repository); // For faster lookups + else + cachedMetadata->setAvailableFromDefaultRepository(true); + + // Refresh also persistent information, the url of the repository may have changed + // from the last fetch. + cachedMetadata->setPersistentRepositoryPath(repository.url()); + + // search for additional repositories that we might need to check + if (cachedMetadata->containsRepositoryUpdates()) { + QDomDocument doc = cachedMetadata->updatesDocument(); + const Status status = parseRepositoryUpdates(doc.documentElement(), result, cachedMetadata); + if (status == XmlDownloadRetry) { + // The repository update may have removed or replaced current repositories, + // clear repository information from cached items and refresh on next fetch run. + resetCacheRepositories(); + return status; + } + } + *refreshed = true; + return XmlDownloadSuccess; + } + // Missing or corrupted files, or compressed repository which takes priority + // over remote repository. We will re-download and uncompress + // the metadata. Remove broken item from the cache. + if (!m_metaFromCache.removeItem(checksum)) { + qCWarning(lcInstallerInstallLog) << m_metaFromCache.errorString(); + return XmlDownloadFailure; + } + return XmlDownloadSuccess; +} + +MetadataJob::Status MetadataJob::findCachedUpdatesFile(const Repository &repository, const QString &fileUrl) +{ + if (repository.xmlChecksum().isEmpty()) + return XmlDownloadFailure; + + Metadata *metadata = m_metaFromCache.itemByChecksum(repository.xmlChecksum()); + if (!metadata) + return XmlDownloadFailure; + + const QString targetPath = metadata->path() + QLatin1Char('/') + scUpdatesXML; + + FileTaskItem cachedMetaTaskItem(fileUrl, targetPath); + cachedMetaTaskItem.insert(TaskRole::UserRole, QVariant::fromValue(repository)); + const FileTaskResult cachedMetaTaskResult(targetPath, repository.xmlChecksum(), cachedMetaTaskItem, false); + + bool isCached = false; + const Status status = refreshCacheItem(cachedMetaTaskResult, repository.xmlChecksum(), &isCached); + if (isCached) + return XmlDownloadSuccess; + else if (status == XmlDownloadRetry) + return XmlDownloadRetry; + else + return XmlDownloadFailure; +} + +MetadataJob::Status MetadataJob::parseRepositoryUpdates(const QDomElement &root, + const FileTaskResult &result, Metadata *metadata) +{ + MetadataJob::Status status = XmlDownloadSuccess; + const QDomNode repositoryUpdate = root.firstChildElement(QLatin1String("RepositoryUpdate")); + if (!repositoryUpdate.isNull()) { + const QMultiHash<QString, QPair<Repository, Repository> > repositoryUpdates + = searchAdditionalRepositories(repositoryUpdate, result, *metadata); + if (!repositoryUpdates.isEmpty()) + status = setAdditionalRepositories(repositoryUpdates, result, *metadata); + } + return status; +} + QSet<Repository> MetadataJob::getRepositories() { QSet<Repository> repositories; - //In the first run, m_metadata is empty. Get always the default repositories - if (m_metaFromDefaultRepositories.isEmpty()) { + //In the first run, get always the default repositories + if (!m_defaultRepositoriesFetched) { repositories = m_core->settings().repositories(); + m_defaultRepositoriesFetched = true; } // Fetch repositories under archive which are selected in UI. // If repository is already fetched, do not fetch it again. - // In updater mode, fetch always all archive repositories to get updates - foreach (RepositoryCategory repositoryCategory, m_core->settings().repositoryCategories()) { - if (m_core->isUpdater() || (repositoryCategory.isEnabled())) { - foreach (Repository repository, repositoryCategory.repositories()) { - QHashIterator<QString, ArchiveMetadata> i(m_fetchedArchive); - bool fetch = true; - while (i.hasNext()) { - i.next(); - ArchiveMetadata metaData = i.value(); - if (repository.url() == metaData.metaData.repository.url()) - fetch = false; - } - if (fetch) - repositories.insert(repository); - } - } + for (const RepositoryCategory &repositoryCategory : m_core->settings().repositoryCategories()) { + if (!repositoryCategory.isEnabled()) + continue; + + for (const Repository &repository : repositoryCategory.repositories()) { + if (!m_fetchedCategorizedRepositories.contains(repository)) + repositories.insert(repository); + } } return repositories; } -void MetadataJob::addFileTaskItem(const QString &source, const QString &target, const Metadata &metadata, +void MetadataJob::addFileTaskItem(const QString &source, const QString &target, Metadata *metadata, const QString &sha1, const QString &packageName) { FileTaskItem item(source, target); QAuthenticator authenticator; - authenticator.setUser(metadata.repository.username()); - authenticator.setPassword(metadata.repository.password()); + authenticator.setUser(metadata->repository().username()); + authenticator.setPassword(metadata->repository().password()); - item.insert(TaskRole::UserRole, metadata.directory); + item.insert(TaskRole::UserRole, metadata->path()); item.insert(TaskRole::Checksum, sha1.toLatin1()); item.insert(TaskRole::Authenticator, QVariant::fromValue(authenticator)); item.insert(TaskRole::Name, packageName); @@ -775,7 +1062,7 @@ bool MetadataJob::parsePackageUpdate(const QDomNodeList &c2, QString &packageNam else if ((element.tagName() == QLatin1String("SHA1")) && testCheckSum) packageHash = element.text(); else { - foreach (QString meta, metaElements) { + foreach (QString meta, scMetaElements) { if (element.tagName() == meta) { metaFound = true; break; @@ -786,10 +1073,10 @@ bool MetadataJob::parsePackageUpdate(const QDomNodeList &c2, QString &packageNam return metaFound; } -QHash<QString, QPair<Repository, Repository> > MetadataJob::searchAdditionalRepositories +QMultiHash<QString, QPair<Repository, Repository> > MetadataJob::searchAdditionalRepositories (const QDomNode &repositoryUpdate, const FileTaskResult &result, const Metadata &metadata) { - QHash<QString, QPair<Repository, Repository> > repositoryUpdates; + QMultiHash<QString, QPair<Repository, Repository> > repositoryUpdates; const QDomNodeList children = repositoryUpdate.toElement().childNodes(); for (int i = 0; i < children.count(); ++i) { const QDomElement el = children.at(i).toElement(); @@ -802,14 +1089,14 @@ QHash<QString, QPair<Repository, Repository> > MetadataJob::searchAdditionalRepo repository.setPassword(el.attribute(QLatin1String("password"))); repository.setDisplayName(el.attribute(QLatin1String("displayname"))); if (ProductKeyCheck::instance()->isValidRepository(repository)) { - repositoryUpdates.insertMulti(action, qMakePair(repository, Repository())); + repositoryUpdates.insert(action, qMakePair(repository, Repository())); qDebug() << "Repository to add:" << repository.displayname(); } } else if (action == QLatin1String("remove")) { // remove possible default repositories using the given server url Repository repository(resolveUrl(result, el.attribute(QLatin1String("url"))), true); repository.setDisplayName(el.attribute(QLatin1String("displayname"))); - repositoryUpdates.insertMulti(action, qMakePair(repository, Repository())); + repositoryUpdates.insert(action, qMakePair(repository, Repository())); qDebug() << "Repository to remove:" << repository.displayname(); } else if (action == QLatin1String("replace")) { @@ -822,27 +1109,27 @@ QHash<QString, QPair<Repository, Repository> > MetadataJob::searchAdditionalRepo if (ProductKeyCheck::instance()->isValidRepository(newRepository)) { // store the new repository and the one old it replaces - repositoryUpdates.insertMulti(action, qMakePair(newRepository, oldRepository)); + repositoryUpdates.insert(action, qMakePair(newRepository, oldRepository)); qDebug() << "Replace repository" << oldRepository.displayname() << "with" << newRepository.displayname(); } } else { qDebug() << "Invalid additional repositories action set in Updates.xml fetched " - "from" << metadata.repository.displayname() << "line:" << el.lineNumber(); + "from" << metadata.repository().displayname() << "line:" << el.lineNumber(); } } } return repositoryUpdates; } -MetadataJob::Status MetadataJob::setAdditionalRepositories(QHash<QString, QPair<Repository, Repository> > repositoryUpdates, +MetadataJob::Status MetadataJob::setAdditionalRepositories(QMultiHash<QString, QPair<Repository, Repository> > repositoryUpdates, const FileTaskResult &result, const Metadata& metadata) { MetadataJob::Status status = XmlDownloadSuccess; Settings &s = m_core->settings(); const QSet<Repository> temporaries = s.temporaryRepositories(); // in case the temp repository introduced something new, we only want that temporary - if (temporaries.contains(metadata.repository)) { + if (temporaries.contains(metadata.repository())) { QSet<Repository> tmpRepositories; typedef QPair<Repository, Repository> RepositoryPair; @@ -858,7 +1145,9 @@ MetadataJob::Status MetadataJob::setAdditionalRepositories(QHash<QString, QPair< if (tmpRepositories.count() > 0) { s.addTemporaryRepositories(tmpRepositories, true); QFile::remove(result.target()); - m_metaFromDefaultRepositories.clear(); + m_defaultRepositoriesFetched = false; + qDeleteAll(m_fetchedMetadata); + m_fetchedMetadata.clear(); status = XmlDownloadRetry; } } else if (s.updateDefaultRepositories(repositoryUpdates) == Settings::UpdatesApplied) { @@ -872,10 +1161,22 @@ MetadataJob::Status MetadataJob::setAdditionalRepositories(QHash<QString, QPair< if (gainedAdminRights) m_core->dropAdminRights(); } - m_metaFromDefaultRepositories.clear(); + m_defaultRepositoriesFetched = false; + qDeleteAll(m_fetchedMetadata); + m_fetchedMetadata.clear(); QFile::remove(result.target()); status = XmlDownloadRetry; } return status; } + +void MetadataJob::setInfoMessage(const QString &message) +{ + m_taskNumber++; + QString metaInformation = message; + if (m_totalTaskCount > 1) + metaInformation = QLatin1String(" %1 %2/%3 ").arg(message).arg(m_taskNumber).arg(m_totalTaskCount); + emit infoMessage(this, metaInformation); + +} } // namespace QInstaller |