Add possibility to compress metadata per repository

Added two new configure options for repogen. --unite-metadata will
create both 'old' style meta 7z, which is one per component, and new
style which is one per repository. --unite-metadata-only will create
only new syntax meta 7z. One meta 7z per repository will significantly
decrease the download time if there are several packages in one
repository.

Change-Id: I651b24e93fdef3efb6253ee9b119ebad3bae4d59
Reviewed-by: Iikka Eklund <iikka.eklund@qt.io>

4 files changed, 76 insertions, 15 deletions

diff --git a/tools/common/repositorygen.cpp b/tools/common/repositorygen.cpp
index 7034d32fb..d47189764 100644
--- a/tools/common/repositorygen.cpp
+++ b/tools/common/repositorygen.cpp
@@ -626,14 +626,17 @@ QHash<QString, QString> QInstallerTools::buildPathToVersionMapping(const Package
 }
 
 static void writeSHA1ToNodeWithName(QDomDocument &doc, QDomNodeList &list, const QByteArray &sha1sum,
-    const QString &nodename)
+    const QString &nodename = QString())
 {
-    qDebug() << "Searching sha1sum node for" << nodename;
+    if (nodename.isEmpty())
+        qDebug() << "Searching sha1sum node.";
+    else
+        qDebug() << "Searching sha1sum node for " << nodename;
     QString sha1Value = QString::fromLatin1(sha1sum.toHex().constData());
     for (int i = 0; i < list.size(); ++i) {
         QDomNode curNode = list.at(i);
         QDomNode nameTag = curNode.firstChildElement(scName);
-        if (!nameTag.isNull() && nameTag.toElement().text() == nodename) {
+        if ((!nameTag.isNull() && nameTag.toElement().text() == nodename) || nodename.isEmpty()) {
             QDomNode sha1Node = curNode.firstChildElement(scSHA1);
             if (!sha1Node.isNull() && sha1Node.hasChildNodes()) {
                 QDomNode sha1NodeChild = sha1Node.firstChild();
@@ -656,7 +659,7 @@ static void writeSHA1ToNodeWithName(QDomDocument &doc, QDomNodeList &list, const
 }
 
 void QInstallerTools::compressMetaDirectories(const QString &repoDir, const QString &baseDir,
-    const QHash<QString, QString> &versionMapping)
+    const QHash<QString, QString> &versionMapping, bool createUnitedMetadata, bool createOnlyUnitedMetadata)
 {
     QDomDocument doc;
     QDomElement root;
@@ -683,19 +686,60 @@ void QInstallerTools::compressMetaDirectories(const QString &repoDir, const QStr
         const QString fn = QLatin1String(versionPrefix.toLatin1() + "meta.7z");
         const QString tmpTarget = repoDir + QLatin1String("/") +fn;
         Lib7z::createArchive(tmpTarget, QStringList() << absPath, Lib7z::TmpFile::No);
+    }
 
-        // remove the files that got compressed
-        QInstaller::removeFiles(absPath, true);
+    QStringList absPaths;
+    if (createUnitedMetadata || createOnlyUnitedMetadata) {
+        QStringList finalTargets;
+        foreach (const QString &i, sub) {
+            QDir sd(dir);
+            sd.cd(i);
+            const QString absPath = sd.absolutePath();
+            finalTargets.append(absPath);
+            absPaths.append(absPath);
+        }
+
+        // Compress all metadata from repository to one single 7z
+        const QString fn = QLatin1String("meta.7z");
+        const QString tmpTarget = repoDir + QLatin1String("/") +fn;
+        Lib7z::createArchive(tmpTarget, finalTargets, Lib7z::TmpFile::No);
 
         QFile tmp(tmpTarget);
         tmp.open(QFile::ReadOnly);
         const QByteArray sha1Sum = QInstaller::calculateHash(&tmp, QCryptographicHash::Sha1);
-        writeSHA1ToNodeWithName(doc, elements, sha1Sum, path);
-        const QString finalTarget = absPath + QLatin1String("/") + fn;
-        if (!tmp.rename(finalTarget)) {
-            throw QInstaller::Error(QString::fromLatin1("Cannot move file \"%1\" to \"%2\".").arg(
-                                        QDir::toNativeSeparators(tmpTarget), QDir::toNativeSeparators(finalTarget)));
+        elements =  doc.elementsByTagName(QLatin1String("Updates"));
+        writeSHA1ToNodeWithName(doc, elements, sha1Sum, QString());
+    }
+
+    if (!createOnlyUnitedMetadata) {
+        foreach (const QString &i, sub) {
+            QDir sd(dir);
+            sd.cd(i);
+            const QString absPath = sd.absolutePath();
+            const QString path = QString(i).remove(baseDir);
+            const QString versionPrefix = versionMapping[path];
+            if (path.isNull())
+                continue;
+            const QString fn = QLatin1String(versionPrefix.toLatin1() + "meta.7z");
+            const QString tmpTarget = repoDir + QLatin1String("/") + fn;
+            Lib7z::createArchive(tmpTarget, QStringList() << absPath, Lib7z::TmpFile::No);
+            // remove the files that got compressed
+            QInstaller::removeFiles(absPath, true);
+            QFile tmp(tmpTarget);
+            tmp.open(QFile::ReadOnly);
+            const QByteArray sha1Sum = QInstaller::calculateHash(&tmp, QCryptographicHash::Sha1);
+            writeSHA1ToNodeWithName(doc, elements, sha1Sum, path);
+            const QString finalTarget = absPath + QLatin1String("/") + fn;
+            if (!tmp.rename(finalTarget)) {
+                throw QInstaller::Error(QString::fromLatin1("Cannot move file \"%1\" to \"%2\".").arg(
+                                            QDir::toNativeSeparators(tmpTarget), QDir::toNativeSeparators(finalTarget)));
+            }
+
         }
+    } else {
+        // remove the files that got compressed
+        foreach (const QString path, absPaths)
+            QInstaller::removeFiles(path, true);
     }
 
     QInstaller::openForWrite(&existingUpdatesXml);
diff --git a/tools/common/repositorygen.h b/tools/common/repositorygen.h
index ead9e15be..c3ae6869f 100644
--- a/tools/common/repositorygen.h
+++ b/tools/common/repositorygen.h
@@ -67,7 +67,7 @@ PackageInfoVector createListOfRepositoryPackages(const QStringList &repositoryDi
 QHash<QString, QString> buildPathToVersionMapping(const PackageInfoVector &info);
 
 void compressMetaDirectories(const QString &repoDir, const QString &baseDir,
-    const QHash<QString, QString> &versionMapping);
+    const QHash<QString, QString> &versionMapping, bool createUnitedMetadata, bool createOnlyUnitedMetadata);
 
 void copyMetaData(const QString &outDir, const QString &dataDir, const PackageInfoVector &packages,
     const QString &appName, const QString& appVersion);
diff --git a/tools/devtool/main.cpp b/tools/devtool/main.cpp
index dbe8f2b09..208ef148e 100644
--- a/tools/devtool/main.cpp
+++ b/tools/devtool/main.cpp
@@ -96,7 +96,7 @@ static QStringList split(int index, const QString &description)
 int main(int argc, char *argv[])
 {
     QCoreApplication app(argc, argv);
-    app.setApplicationVersion(QLatin1String("1.0.0"));
+    app.setApplicationVersion(QLatin1String("2.0.0"));
 
     QCommandLineParser parser;
     QCommandLineOption help = parser.addHelpOption();
diff --git a/tools/repogen/repogen.cpp b/tools/repogen/repogen.cpp
index e785ffd04..2d16fa720 100644
--- a/tools/repogen/repogen.cpp
+++ b/tools/repogen/repogen.cpp
@@ -68,6 +68,13 @@ static void printUsage()
 
     std::cout << "  -v|--verbose              Verbose output" << std::endl;
 
+    std::cout << "  --unite-metadata          Combine all metadata into one 7z. This speeds up metadata " << std::endl;
+    std::cout << "                            download phase. Creates also metadata 7z per component for " << std::endl;
+    std::cout << "                            backward compatibility." << std::endl;
+
+    std::cout << "  --unite-metadata-only     Combine all metadata into one 7z. This speeds up metadata " << std::endl;
+    std::cout << "                            download phase." << std::endl;
+
     std::cout << std::endl;
     std::cout << "Example:" << std::endl;
     std::cout << "  " << appName << " -p ../examples/packages repository/"
@@ -99,6 +106,8 @@ int main(int argc, char** argv)
         QInstallerTools::FilterType filterType = QInstallerTools::Exclude;
         bool remove = false;
         bool updateExistingRepositoryWithNewComponents = false;
+        bool createUnitedMetadata = false;
+        bool createOnlyUnitedMetadata = false;
 
         //TODO: use a for loop without removing values from args like it is in binarycreator.cpp
         //for (QStringList::const_iterator it = args.begin(); it != args.end(); ++it) {
@@ -176,7 +185,14 @@ int main(int argc, char** argv)
             } else if (args.first() == QLatin1String("-r") || args.first() == QLatin1String("--remove")) {
                 remove = true;
                 args.removeFirst();
-            } else {
+            } else if (args.first() == QLatin1String("--unite-metadata")) {
+                createUnitedMetadata = true;
+                args.removeFirst();
+            } else if (args.first() == QLatin1String("--unite-metadata-only")) {
+                createOnlyUnitedMetadata = true;
+                args.removeFirst();
+            }
+            else {
                 printUsage();
                 return 1;
             }
@@ -281,7 +297,8 @@ int main(int argc, char** argv)
         QInstallerTools::copyComponentData(directories, repositoryDir, &packages);
         QInstallerTools::copyMetaData(tmpMetaDir, repositoryDir, packages, QLatin1String("{AnyApplication}"),
             QLatin1String(QUOTE(IFW_REPOSITORY_FORMAT_VERSION)));
-        QInstallerTools::compressMetaDirectories(tmpMetaDir, tmpMetaDir, pathToVersionMapping);
+        QInstallerTools::compressMetaDirectories(tmpMetaDir, tmpMetaDir, pathToVersionMapping,
+                                                 createUnitedMetadata, createOnlyUnitedMetadata);
 
         QDirIterator it(repositoryDir, QStringList(QLatin1String("Updates*.xml")), QDir::Files | QDir::CaseSensitive);
         while (it.hasNext()) {