diff options
author | Katja Marttila <katja.marttila@qt.io> | 2019-10-30 09:50:05 +0200 |
---|---|---|
committer | Katja Marttila <katja.marttila@qt.io> | 2020-03-24 07:38:56 +0200 |
commit | 14a2502c7781a7e8036912767eea990a92dd560c (patch) | |
tree | 0e5137943ee8ebd56892a21f41cde3486ab2c8ba /tools | |
parent | 4d7c5d092e95f2eb2113780971027e0adfa41440 (diff) |
Add possibility to compress metadata per repository
Added two new configure options for repogen. --unite-metadata will
create both 'old' style meta 7z, which is one per component, and new
style which is one per repository. --unite-metadata-only will create
only new syntax meta 7z. One meta 7z per repository will significantly
decrease the download time if there are several packages in one
repository.
Change-Id: I651b24e93fdef3efb6253ee9b119ebad3bae4d59
Reviewed-by: Iikka Eklund <iikka.eklund@qt.io>
Diffstat (limited to 'tools')
-rw-r--r-- | tools/common/repositorygen.cpp | 66 | ||||
-rw-r--r-- | tools/common/repositorygen.h | 2 | ||||
-rw-r--r-- | tools/devtool/main.cpp | 2 | ||||
-rw-r--r-- | tools/repogen/repogen.cpp | 21 |
4 files changed, 76 insertions, 15 deletions
diff --git a/tools/common/repositorygen.cpp b/tools/common/repositorygen.cpp index 7034d32fb..d47189764 100644 --- a/tools/common/repositorygen.cpp +++ b/tools/common/repositorygen.cpp @@ -626,14 +626,17 @@ QHash<QString, QString> QInstallerTools::buildPathToVersionMapping(const Package } static void writeSHA1ToNodeWithName(QDomDocument &doc, QDomNodeList &list, const QByteArray &sha1sum, - const QString &nodename) + const QString &nodename = QString()) { - qDebug() << "Searching sha1sum node for" << nodename; + if (nodename.isEmpty()) + qDebug() << "Searching sha1sum node."; + else + qDebug() << "Searching sha1sum node for " << nodename; QString sha1Value = QString::fromLatin1(sha1sum.toHex().constData()); for (int i = 0; i < list.size(); ++i) { QDomNode curNode = list.at(i); QDomNode nameTag = curNode.firstChildElement(scName); - if (!nameTag.isNull() && nameTag.toElement().text() == nodename) { + if ((!nameTag.isNull() && nameTag.toElement().text() == nodename) || nodename.isEmpty()) { QDomNode sha1Node = curNode.firstChildElement(scSHA1); if (!sha1Node.isNull() && sha1Node.hasChildNodes()) { QDomNode sha1NodeChild = sha1Node.firstChild(); @@ -656,7 +659,7 @@ static void writeSHA1ToNodeWithName(QDomDocument &doc, QDomNodeList &list, const } void QInstallerTools::compressMetaDirectories(const QString &repoDir, const QString &baseDir, - const QHash<QString, QString> &versionMapping) + const QHash<QString, QString> &versionMapping, bool createUnitedMetadata, bool createOnlyUnitedMetadata) { QDomDocument doc; QDomElement root; @@ -683,19 +686,60 @@ void QInstallerTools::compressMetaDirectories(const QString &repoDir, const QStr const QString fn = QLatin1String(versionPrefix.toLatin1() + "meta.7z"); const QString tmpTarget = repoDir + QLatin1String("/") +fn; Lib7z::createArchive(tmpTarget, QStringList() << absPath, Lib7z::TmpFile::No); + } - // remove the files that got compressed - QInstaller::removeFiles(absPath, true); + QStringList absPaths; + if (createUnitedMetadata || createOnlyUnitedMetadata) { + QStringList finalTargets; + foreach (const QString &i, sub) { + QDir sd(dir); + sd.cd(i); + const QString absPath = sd.absolutePath(); + finalTargets.append(absPath); + absPaths.append(absPath); + } + + // Compress all metadata from repository to one single 7z + const QString fn = QLatin1String("meta.7z"); + const QString tmpTarget = repoDir + QLatin1String("/") +fn; + Lib7z::createArchive(tmpTarget, finalTargets, Lib7z::TmpFile::No); QFile tmp(tmpTarget); tmp.open(QFile::ReadOnly); const QByteArray sha1Sum = QInstaller::calculateHash(&tmp, QCryptographicHash::Sha1); - writeSHA1ToNodeWithName(doc, elements, sha1Sum, path); - const QString finalTarget = absPath + QLatin1String("/") + fn; - if (!tmp.rename(finalTarget)) { - throw QInstaller::Error(QString::fromLatin1("Cannot move file \"%1\" to \"%2\".").arg( - QDir::toNativeSeparators(tmpTarget), QDir::toNativeSeparators(finalTarget))); + elements = doc.elementsByTagName(QLatin1String("Updates")); + writeSHA1ToNodeWithName(doc, elements, sha1Sum, QString()); + } + + if (!createOnlyUnitedMetadata) { + foreach (const QString &i, sub) { + QDir sd(dir); + sd.cd(i); + const QString absPath = sd.absolutePath(); + const QString path = QString(i).remove(baseDir); + const QString versionPrefix = versionMapping[path]; + if (path.isNull()) + continue; + const QString fn = QLatin1String(versionPrefix.toLatin1() + "meta.7z"); + const QString tmpTarget = repoDir + QLatin1String("/") + fn; + Lib7z::createArchive(tmpTarget, QStringList() << absPath, Lib7z::TmpFile::No); + // remove the files that got compressed + QInstaller::removeFiles(absPath, true); + QFile tmp(tmpTarget); + tmp.open(QFile::ReadOnly); + const QByteArray sha1Sum = QInstaller::calculateHash(&tmp, QCryptographicHash::Sha1); + writeSHA1ToNodeWithName(doc, elements, sha1Sum, path); + const QString finalTarget = absPath + QLatin1String("/") + fn; + if (!tmp.rename(finalTarget)) { + throw QInstaller::Error(QString::fromLatin1("Cannot move file \"%1\" to \"%2\".").arg( + QDir::toNativeSeparators(tmpTarget), QDir::toNativeSeparators(finalTarget))); + } + } + } else { + // remove the files that got compressed + foreach (const QString path, absPaths) + QInstaller::removeFiles(path, true); } QInstaller::openForWrite(&existingUpdatesXml); diff --git a/tools/common/repositorygen.h b/tools/common/repositorygen.h index ead9e15be..c3ae6869f 100644 --- a/tools/common/repositorygen.h +++ b/tools/common/repositorygen.h @@ -67,7 +67,7 @@ PackageInfoVector createListOfRepositoryPackages(const QStringList &repositoryDi QHash<QString, QString> buildPathToVersionMapping(const PackageInfoVector &info); void compressMetaDirectories(const QString &repoDir, const QString &baseDir, - const QHash<QString, QString> &versionMapping); + const QHash<QString, QString> &versionMapping, bool createUnitedMetadata, bool createOnlyUnitedMetadata); void copyMetaData(const QString &outDir, const QString &dataDir, const PackageInfoVector &packages, const QString &appName, const QString& appVersion); diff --git a/tools/devtool/main.cpp b/tools/devtool/main.cpp index dbe8f2b09..208ef148e 100644 --- a/tools/devtool/main.cpp +++ b/tools/devtool/main.cpp @@ -96,7 +96,7 @@ static QStringList split(int index, const QString &description) int main(int argc, char *argv[]) { QCoreApplication app(argc, argv); - app.setApplicationVersion(QLatin1String("1.0.0")); + app.setApplicationVersion(QLatin1String("2.0.0")); QCommandLineParser parser; QCommandLineOption help = parser.addHelpOption(); diff --git a/tools/repogen/repogen.cpp b/tools/repogen/repogen.cpp index e785ffd04..2d16fa720 100644 --- a/tools/repogen/repogen.cpp +++ b/tools/repogen/repogen.cpp @@ -68,6 +68,13 @@ static void printUsage() std::cout << " -v|--verbose Verbose output" << std::endl; + std::cout << " --unite-metadata Combine all metadata into one 7z. This speeds up metadata " << std::endl; + std::cout << " download phase. Creates also metadata 7z per component for " << std::endl; + std::cout << " backward compatibility." << std::endl; + + std::cout << " --unite-metadata-only Combine all metadata into one 7z. This speeds up metadata " << std::endl; + std::cout << " download phase." << std::endl; + std::cout << std::endl; std::cout << "Example:" << std::endl; std::cout << " " << appName << " -p ../examples/packages repository/" @@ -99,6 +106,8 @@ int main(int argc, char** argv) QInstallerTools::FilterType filterType = QInstallerTools::Exclude; bool remove = false; bool updateExistingRepositoryWithNewComponents = false; + bool createUnitedMetadata = false; + bool createOnlyUnitedMetadata = false; //TODO: use a for loop without removing values from args like it is in binarycreator.cpp //for (QStringList::const_iterator it = args.begin(); it != args.end(); ++it) { @@ -176,7 +185,14 @@ int main(int argc, char** argv) } else if (args.first() == QLatin1String("-r") || args.first() == QLatin1String("--remove")) { remove = true; args.removeFirst(); - } else { + } else if (args.first() == QLatin1String("--unite-metadata")) { + createUnitedMetadata = true; + args.removeFirst(); + } else if (args.first() == QLatin1String("--unite-metadata-only")) { + createOnlyUnitedMetadata = true; + args.removeFirst(); + } + else { printUsage(); return 1; } @@ -281,7 +297,8 @@ int main(int argc, char** argv) QInstallerTools::copyComponentData(directories, repositoryDir, &packages); QInstallerTools::copyMetaData(tmpMetaDir, repositoryDir, packages, QLatin1String("{AnyApplication}"), QLatin1String(QUOTE(IFW_REPOSITORY_FORMAT_VERSION))); - QInstallerTools::compressMetaDirectories(tmpMetaDir, tmpMetaDir, pathToVersionMapping); + QInstallerTools::compressMetaDirectories(tmpMetaDir, tmpMetaDir, pathToVersionMapping, + createUnitedMetadata, createOnlyUnitedMetadata); QDirIterator it(repositoryDir, QStringList(QLatin1String("Updates*.xml")), QDir::Files | QDir::CaseSensitive); while (it.hasNext()) { |