Repogen: Fix options to work with unite metadata

Fixed regogen 'update' and 'repository' options to work with unite
metadata.
--update-new-components with unify metadata is prevented.
Moved --update-new-components functionality from repogen.cpp to
QInstallerTools so it can be accessed from tests.

Created tests for repogen.

Task-number: QTIFW-1943
Change-Id: I12389f4747154a0f6f51b4f34f630103c2cfef04
Reviewed-by: Arttu Tarkiainen <arttu.tarkiainen@qt.io>

6 files changed, 137 insertions, 80 deletions

diff --git a/tools/binarycreator/binarycreator.cpp b/tools/binarycreator/binarycreator.cpp
index 6caf3ff1f..f6662615e 100644
--- a/tools/binarycreator/binarycreator.cpp
+++ b/tools/binarycreator/binarycreator.cpp
@@ -986,7 +986,7 @@ int main(int argc, char **argv)
 
         // 3; copy the meta data of the available packages, generate Updates.xml
         QInstallerTools::copyMetaData(tmpMetaDir, tmpRepoDir, packages, settings
-            .applicationName(), settings.version());
+            .applicationName(), settings.version(), QStringList());
 
         // 4; copy the configuration file and and icons etc.
         copyConfigData(configFile, tmpMetaDir + QLatin1String("/installer-config"));
diff --git a/tools/common/repositorygen.cpp b/tools/common/repositorygen.cpp
index 4215f127d..6f17ad5ac 100644
--- a/tools/common/repositorygen.cpp
+++ b/tools/common/repositorygen.cpp
@@ -1,6 +1,6 @@
 /**************************************************************************
 **
-** Copyright (C) 2017 The Qt Company Ltd.
+** Copyright (C) 2020 The Qt Company Ltd.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the Qt Installer Framework.
@@ -93,7 +93,7 @@ void QInstallerTools::copyWithException(const QString &source, const QString &ta
             (targetFileInfo.exists() ? QLatin1String("Target already exist.") : sourceFile.errorString())));
     }
 
-    qDebug() << "done.\n";
+    qDebug() << "done.";
 }
 
 static QStringList copyFilesFromNode(const QString &parentNode, const QString &childNode, const QString &attr,
@@ -125,12 +125,14 @@ static QStringList copyFilesFromNode(const QString &parentNode, const QString &c
 }
 
 void QInstallerTools::copyMetaData(const QString &_targetDir, const QString &metaDataDir,
-    const PackageInfoVector &packages, const QString &appName, const QString &appVersion)
+    const PackageInfoVector &packages, const QString &appName, const QString &appVersion,
+    const QStringList &uniteMetadatas)
 {
     const QString targetDir = makePathAbsolute(_targetDir);
     if (!QFile::exists(targetDir))
         QInstaller::mkpath(targetDir);
 
+    bool componentMetaExtracted = false;
     QDomDocument doc;
     QDomElement root;
     QFile existingUpdatesXml(QFileInfo(metaDataDir, QLatin1String("Updates.xml")).absoluteFilePath());
@@ -377,9 +379,12 @@ void QInstallerTools::copyMetaData(const QString &_targetDir, const QString &met
             }
         } else {
             // Extract metadata from archive
-            QFile metaFile(info.metaFile);
-            QInstaller::openForRead(&metaFile);
-            Lib7z::extractArchive(&metaFile, targetDir);
+            if (!info.metaFile.isEmpty()){
+                QFile metaFile(info.metaFile);
+                QInstaller::openForRead(&metaFile);
+                Lib7z::extractArchive(&metaFile, targetDir);
+                componentMetaExtracted = true;
+            }
 
             // Restore "PackageUpdate" node;
             QDomDocument update;
@@ -391,6 +396,14 @@ void QInstallerTools::copyMetaData(const QString &_targetDir, const QString &met
         }
     }
 
+    if (!componentMetaExtracted) {
+        foreach (const QString uniteMetadata, uniteMetadatas) {
+            QFile metaFile(QFileInfo(metaDataDir, uniteMetadata).absoluteFilePath());
+            QInstaller::openForRead(&metaFile);
+            Lib7z::extractArchive(&metaFile, targetDir);
+        }
+    }
+
     doc.appendChild(root);
 
     QFile targetUpdatesXml(targetDir + QLatin1String("/Updates.xml"));
@@ -401,7 +414,7 @@ void QInstallerTools::copyMetaData(const QString &_targetDir, const QString &met
 PackageInfoVector QInstallerTools::createListOfPackages(const QStringList &packagesDirectories,
     QStringList *packagesToFilter, FilterType filterType)
 {
-    qDebug() << "\nCollecting information about available packages...";
+    qDebug() << "Collecting information about available packages...";
 
     bool ignoreInvalidPackages = qApp->arguments().contains(QString::fromLatin1("--ignore-invalid-packages"));
 
@@ -583,16 +596,20 @@ PackageInfoVector QInstallerTools::createListOfRepositoryPackages(const QStringL
                     continue;
 
                 info.directory = QString::fromLatin1("%1/%2").arg(it->filePath(), info.name);
-                info.metaFile = QString::fromLatin1("%1/%3%2").arg(info.directory,
-                    QString::fromLatin1("meta.7z"), info.version);
+                const QDomElement sha1 = el.firstChildElement(QInstaller::scSHA1);
+                if (!sha1.isNull()) {
+                    info.metaFile = QString::fromLatin1("%1/%3%2").arg(info.directory,
+                        QString::fromLatin1("meta.7z"), info.version);
+                }
 
                 const QDomNodeList c2 = el.childNodes();
                 for (int j = 0; j < c2.count(); ++j) {
-                    if (c2.at(j).toElement().tagName() == QInstaller::scDependencies)
-                        info.dependencies = c2.at(j).toElement().text()
+                    const QDomElement c2Element = c2.at(j).toElement();
+                    if (c2Element.tagName() == QInstaller::scDependencies)
+                        info.dependencies = c2Element.text()
                             .split(QInstaller::commaRegExp(), QString::SkipEmptyParts);
-                    else if (c2.at(j).toElement().tagName() == QInstaller::scDownloadableArchives) {
-                        QStringList names = c2.at(j).toElement().text()
+                    else if (c2Element.tagName() == QInstaller::scDownloadableArchives) {
+                        QStringList names = c2Element.text()
                             .split(QInstaller::commaRegExp(), QString::SkipEmptyParts);
                         foreach (const QString &name, names) {
                             info.copiedFiles.append(QString::fromLatin1("%1/%3%2").arg(info.directory,
@@ -626,18 +643,21 @@ QHash<QString, QString> QInstallerTools::buildPathToVersionMapping(const Package
 }
 
 static void writeSHA1ToNodeWithName(QDomDocument &doc, QDomNodeList &list, const QByteArray &sha1sum,
-    const QString &nodename = QString(), const QString &metadataName = QString())
+    const QString &nodename = QString())
 {
     if (nodename.isEmpty())
-        qDebug() << "Searching sha1sum node.";
+        qDebug() << "Writing sha1sum node.";
     else
-        qDebug() << "Searching sha1sum node for " << nodename;
+        qDebug() << "Searching sha1sum node for" << nodename;
     QString sha1Value = QString::fromLatin1(sha1sum.toHex().constData());
     for (int i = 0; i < list.size(); ++i) {
         QDomNode curNode = list.at(i);
         QDomNode nameTag = curNode.firstChildElement(scName);
         if ((!nameTag.isNull() && nameTag.toElement().text() == nodename) || nodename.isEmpty()) {
             QDomNode sha1Node = curNode.firstChildElement(scSHA1);
+            QDomNode newSha1Node = doc.createElement(scSHA1);
+            newSha1Node.appendChild(doc.createTextNode(sha1Value));
+
             if (!sha1Node.isNull() && sha1Node.hasChildNodes()) {
                 QDomNode sha1NodeChild = sha1Node.firstChild();
                 QString sha1OldValue = sha1NodeChild.nodeValue();
@@ -648,19 +668,12 @@ static void writeSHA1ToNodeWithName(QDomDocument &doc, QDomNodeList &list, const
                     qDebug() << "- clearing the old sha1sum" << sha1OldValue;
                     sha1Node.removeChild(sha1NodeChild);
                 }
-            } else {
-                sha1Node = doc.createElement(scSHA1);
-                //Create also unique metadata name
-                if (!metadataName.isEmpty()) {
-                    qDebug() << "Writing the metadata node with name " << metadataName;
-                    QDomNode metaName = doc.createElement(QLatin1String("MetadataName"));
-                    metaName.appendChild(doc.createTextNode(metadataName));
-                    curNode.appendChild(metaName);
-                }
             }
+            if (sha1Node.isNull())
+                curNode.appendChild(newSha1Node);
+            else
+                curNode.replaceChild(newSha1Node, sha1Node);
             qDebug() << "- writing the sha1sum" << sha1Value;
-            sha1Node.appendChild(doc.createTextNode(sha1Value));
-            curNode.appendChild(sha1Node);
         }
     }
 }
@@ -721,7 +734,21 @@ QStringList QInstallerTools::unifyMetadata(const QStringList &entryList, const Q
     tmp.open(QFile::ReadOnly);
     const QByteArray sha1Sum = QInstaller::calculateHash(&tmp, QCryptographicHash::Sha1);
     QDomNodeList elements =  doc.elementsByTagName(QLatin1String("Updates"));
-    writeSHA1ToNodeWithName(doc, elements, sha1Sum, QString(), metadataFilename);
+    writeSHA1ToNodeWithName(doc, elements, sha1Sum, QString());
+
+    qDebug() << "Updating the metadata node with name " << metadataFilename;
+    if (elements.count() > 0) {
+        QDomNode node = elements.at(0);
+        QDomNode nameTag = node.firstChildElement(QLatin1String("MetadataName"));
+
+        QDomNode newNodeTag = doc.createElement(QLatin1String("MetadataName"));
+        newNodeTag.appendChild(doc.createTextNode(metadataFilename));
+
+        if (nameTag.isNull())
+            node.appendChild(newNodeTag);
+        else
+            node.replaceChild(newNodeTag, nameTag);
+    }
     return absPaths;
 }
 
@@ -852,3 +879,48 @@ void QInstallerTools::copyComponentData(const QStringList &packageDirs, const QS
         }
     }
 }
+
+void QInstallerTools::filterNewComponents(const QString &repositoryDir, QInstallerTools::PackageInfoVector &packages)
+{
+    QDomDocument doc;
+    QFile file(repositoryDir + QLatin1String("/Updates.xml"));
+    if (file.open(QFile::ReadOnly) && doc.setContent(&file)) {
+        const QDomElement root = doc.documentElement();
+        if (root.tagName() != QLatin1String("Updates")) {
+            throw QInstaller::Error(QCoreApplication::translate("QInstaller",
+                "Invalid content in \"%1\".").arg(QDir::toNativeSeparators(file.fileName())));
+        }
+        file.close(); // close the file, we read the content already
+
+        // read the already existing updates xml content
+        const QDomNodeList children = root.childNodes();
+        QHash <QString, QInstallerTools::PackageInfo> hash;
+        for (int i = 0; i < children.count(); ++i) {
+            const QDomElement el = children.at(i).toElement();
+            if ((!el.isNull()) && (el.tagName() == QLatin1String("PackageUpdate"))) {
+                QInstallerTools::PackageInfo info;
+                const QDomNodeList c2 = el.childNodes();
+                for (int j = 0; j < c2.count(); ++j) {
+                    const QDomElement c2Element = c2.at(j).toElement();
+                    if (c2Element.tagName() == scName)
+                        info.name = c2Element.text();
+                    else if (c2Element.tagName() == scVersion)
+                        info.version = c2Element.text();
+                }
+                hash.insert(info.name, info);
+            }
+        }
+
+        // remove all components that have no update (decision based on the version tag)
+        for (int i = packages.count() - 1; i >= 0; --i) {
+            const QInstallerTools::PackageInfo info = packages.at(i);
+
+            // check if component already exists & version did not change
+            if (hash.contains(info.name) && KDUpdater::compareVersion(info.version, hash.value(info.name).version) < 1) {
+                packages.remove(i); // the version did not change, no need to update the component
+                continue;
+            }
+            qDebug() << "Update component" << info.name << "in"<< repositoryDir << ".";
+        }
+    }
+}
diff --git a/tools/common/repositorygen.h b/tools/common/repositorygen.h
index b6317e9b7..eb2d23275 100644
--- a/tools/common/repositorygen.h
+++ b/tools/common/repositorygen.h
@@ -75,9 +75,11 @@ void splitMetadata(const QStringList &entryList, const QString &repoDir, QDomDoc
                    const QHash<QString, QString> &versionMapping);
 
 void copyMetaData(const QString &outDir, const QString &dataDir, const PackageInfoVector &packages,
-    const QString &appName, const QString& appVersion);
+    const QString &appName, const QString& appVersion, const QStringList &uniteMetadatas);
 void copyComponentData(const QStringList &packageDir, const QString &repoDir, PackageInfoVector *const infos);
 
+void filterNewComponents(const QString &repositoryDir, QInstallerTools::PackageInfoVector &packages);
+
 
 } // namespace QInstallerTools
 
diff --git a/tools/repogen/repogen.cpp b/tools/repogen/repogen.cpp
index 236b0b901..76d4f676a 100644
--- a/tools/repogen/repogen.cpp
+++ b/tools/repogen/repogen.cpp
@@ -150,6 +150,7 @@ int main(int argc, char** argv)
             } else if (args.first() == QLatin1String("--update-new-components")) {
                 args.removeFirst();
                 updateExistingRepositoryWithNewComponents = true;
+                createUnifiedMetadata = false;
             } else if (args.first() == QLatin1String("-p") || args.first() == QLatin1String("--packages")) {
                 args.removeFirst();
                 if (args.isEmpty()) {
@@ -215,6 +216,17 @@ int main(int argc, char** argv)
         if (remove)
             QInstaller::removeDirectory(repositoryDir);
 
+        if (updateExistingRepositoryWithNewComponents) {
+            QStringList meta7z = QDir(repositoryDir).entryList(QStringList()
+                << QLatin1String("*_meta.7z"), QDir::Files);
+            if (!meta7z.isEmpty()) {
+                throw QInstaller::Error(QCoreApplication::translate("QInstaller",
+                    "Cannot update \"%1\" with --update-new-components. Use --update instead. "
+                    "Currently it is not possible to update partial components inside one 7z.")
+                    .arg(meta7z.join(QLatin1Char(','))));
+            }
+        }
+
         if (!update && QFile::exists(repositoryDir) && !QDir(repositoryDir).entryList(
             QDir::AllEntries | QDir::NoDotAndDotDot).isEmpty()) {
 
@@ -233,53 +245,12 @@ int main(int argc, char** argv)
         packages.append(preparedPackages);
 
         if (updateExistingRepositoryWithNewComponents) {
-            QDomDocument doc;
-            QFile file(repositoryDir + QLatin1String("/Updates.xml"));
-            if (file.open(QFile::ReadOnly) && doc.setContent(&file)) {
-                const QDomElement root = doc.documentElement();
-                if (root.tagName() != QLatin1String("Updates")) {
-                    throw QInstaller::Error(QCoreApplication::translate("QInstaller",
-                        "Invalid content in \"%1\".").arg(QDir::toNativeSeparators(file.fileName())));
-                }
-                file.close(); // close the file, we read the content already
-
-                // read the already existing updates xml content
-                const QDomNodeList children = root.childNodes();
-                QHash <QString, QInstallerTools::PackageInfo> hash;
-                for (int i = 0; i < children.count(); ++i) {
-                    const QDomElement el = children.at(i).toElement();
-                    if ((!el.isNull()) && (el.tagName() == QLatin1String("PackageUpdate"))) {
-                        QInstallerTools::PackageInfo info;
-                        const QDomNodeList c2 = el.childNodes();
-                        for (int j = 0; j < c2.count(); ++j) {
-                            if (c2.at(j).toElement().tagName() == scName)
-                                info.name = c2.at(j).toElement().text();
-                            else if (c2.at(j).toElement().tagName() == scVersion)
-                                info.version = c2.at(j).toElement().text();
-                        }
-                        hash.insert(info.name, info);
-                    }
-                }
-
-                // remove all components that have no update (decision based on the version tag)
-                for (int i = packages.count() - 1; i >= 0; --i) {
-                    const QInstallerTools::PackageInfo info = packages.at(i);
-
-                    // check if component already exists & version did not change
-                    if (hash.contains(info.name) && KDUpdater::compareVersion(info.version, hash.value(info.name).version) < 1) {
-                        packages.remove(i); // the version did not change, no need to update the component
-                        continue;
-                    }
-                    std::cout << QString::fromLatin1("Update component \"%1\" in \"%2\".")
-                        .arg(info.name, repositoryDir) << std::endl;
-                }
-
-                if (packages.isEmpty()) {
-                    std::cout << QString::fromLatin1("Cannot find new components to update \"%1\".")
-                        .arg(repositoryDir) << std::endl;
-                    return EXIT_SUCCESS;
-                }
-            }
+             QInstallerTools::filterNewComponents(repositoryDir, packages);
+             if (packages.isEmpty()) {
+                 std::cout << QString::fromLatin1("Cannot find new components to update \"%1\".")
+                     .arg(repositoryDir) << std::endl;
+                 return EXIT_SUCCESS;
+             }
         }
 
         QHash<QString, QString> pathToVersionMapping = QInstallerTools::buildPathToVersionMapping(packages);
@@ -296,13 +267,23 @@ int main(int argc, char** argv)
         QStringList directories;
         directories.append(packagesDirectories);
         directories.append(repositoryDirectories);
+        QStringList unite7zFiles;
+        foreach (const QString &repositoryDirectory, repositoryDirectories) {
+            QDirIterator it(repositoryDirectory, QStringList(QLatin1String("*_meta.7z"))
+                            , QDir::Files | QDir::CaseSensitive);
+            while (it.hasNext()) {
+                it.next();
+                unite7zFiles.append(it.fileInfo().absoluteFilePath());
+            }
+        }
         QInstallerTools::copyComponentData(directories, repositoryDir, &packages);
         QInstallerTools::copyMetaData(tmpMetaDir, repositoryDir, packages, QLatin1String("{AnyApplication}"),
-            QLatin1String(QUOTE(IFW_REPOSITORY_FORMAT_VERSION)));
+            QLatin1String(QUOTE(IFW_REPOSITORY_FORMAT_VERSION)), unite7zFiles);
         QInstallerTools::compressMetaDirectories(tmpMetaDir, tmpMetaDir, pathToVersionMapping,
                                                  createComponentMetadata, createUnifiedMetadata);
 
-        QDirIterator it(repositoryDir, QStringList(QLatin1String("Updates*.xml")), QDir::Files | QDir::CaseSensitive);
+        QDirIterator it(repositoryDir, QStringList(QLatin1String("Updates*.xml"))
+                        << QLatin1String("*_meta.7z"), QDir::Files | QDir::CaseSensitive);
         while (it.hasNext()) {
             it.next();
             QFile::remove(it.fileInfo().absoluteFilePath());
diff --git a/tools/repogen/repogen.pri b/tools/repogen/repogen.pri
new file mode 100644
index 000000000..319b137ff
--- /dev/null
+++ b/tools/repogen/repogen.pri
@@ -0,0 +1 @@
+INCLUDEPATH += $$PWD/../common
diff --git a/tools/repogen/repogen.pro b/tools/repogen/repogen.pro
index f415a5988..016a8fe8e 100644
--- a/tools/repogen/repogen.pro
+++ b/tools/repogen/repogen.pro
@@ -1,7 +1,8 @@
 TEMPLATE = app
 TARGET = repogen
-INCLUDEPATH += . .. ../common
 
+INCLUDEPATH += . ..
+include(repogen.pri)
 include(../../installerfw.pri)
 
 QT -= gui