Fix install fail if there is huge amount of downloadable packages

In some machines, installer does not manage to download meta files if
there are huge amount of downloadable packages. It seems that
QtConcurrent is not always able to handle so many download tasks.
Divided the downloadable tasks into size of 1000 to keep the
QtConcurrent running.

Change-Id: I3ba110e2cc565251e540349a6b073245280a6fbe
Reviewed-by: Ville Voutilainen <ville.voutilainen@qt.io>

2 files changed, 51 insertions, 20 deletions

diff --git a/src/libs/installer/metadatajob.cpp b/src/libs/installer/metadatajob.cpp
index 0c674e313..9e1e9f5a6 100644
--- a/src/libs/installer/metadatajob.cpp
+++ b/src/libs/installer/metadatajob.cpp
@@ -52,6 +52,8 @@ MetadataJob::MetadataJob(QObject *parent)
     : Job(parent)
     , m_core(0)
     , m_addCompressedPackages(false)
+    , m_downloadableChunkSize(1000)
+    , m_taskNumber(0)
 {
     setCapabilities(Cancelable);
     connect(&m_xmlTask, &QFutureWatcherBase::finished, this, &MetadataJob::xmlTaskFinished);
@@ -325,12 +327,7 @@ void MetadataJob::xmlTaskFinished()
         return;
 
     if (status == XmlDownloadSuccess) {
-        setProcessedAmount(0);
-        DownloadFileTask *const metadataTask = new DownloadFileTask(m_packages);
-        metadataTask->setProxyFactory(m_core->proxyFactory());
-        m_metadataTask.setFuture(QtConcurrent::run(&DownloadFileTask::doTask, metadataTask));
-        setProgressTotalAmount(100);
-        emit infoMessage(this, tr("Retrieving meta information from remote repository..."));
+        fetchMetaDataPackages();
     } else if (status == XmlDownloadRetry) {
         QMetaObject::invokeMethod(this, "doStart", Qt::QueuedConnection);
     } else {
@@ -379,21 +376,23 @@ void MetadataJob::metadataTaskFinished()
 {
     try {
         m_metadataTask.waitForFinished();
-        QFuture<FileTaskResult> future = m_metadataTask.future();
-        if (future.resultCount() > 0) {
-            emit infoMessage(this, tr("Extracting meta information..."));
-            foreach (const FileTaskResult &result, future.results()) {
-                const FileTaskItem item = result.value(TaskRole::TaskItem).value<FileTaskItem>();
-                UnzipArchiveTask *task = new UnzipArchiveTask(result.target(),
-                    item.value(TaskRole::UserRole).toString());
-
-                QFutureWatcher<void> *watcher = new QFutureWatcher<void>();
-                m_unzipTasks.insert(watcher, qobject_cast<QObject*> (task));
-                connect(watcher, &QFutureWatcherBase::finished, this, &MetadataJob::unzipTaskFinished);
-                watcher->setFuture(QtConcurrent::run(&UnzipArchiveTask::doTask, task));
+        m_metadataResult.append(m_metadataTask.future().results());
+        if (!fetchMetaDataPackages()) {
+            if (m_metadataResult.count() > 0) {
+                emit infoMessage(this, tr("Extracting meta information..."));
+                foreach (const FileTaskResult &result, m_metadataResult) {
+                    const FileTaskItem item = result.value(TaskRole::TaskItem).value<FileTaskItem>();
+                    UnzipArchiveTask *task = new UnzipArchiveTask(result.target(),
+                        item.value(TaskRole::UserRole).toString());
+
+                    QFutureWatcher<void> *watcher = new QFutureWatcher<void>();
+                    m_unzipTasks.insert(watcher, qobject_cast<QObject*> (task));
+                    connect(watcher, &QFutureWatcherBase::finished, this, &MetadataJob::unzipTaskFinished);
+                    watcher->setFuture(QtConcurrent::run(&UnzipArchiveTask::doTask, task));
+                }
+            } else {
+                emitFinished();
             }
-        } else {
-            emitFinished();
         }
     } catch (const TaskException &e) {
         reset();
@@ -410,6 +409,30 @@ void MetadataJob::metadataTaskFinished()
 
 // -- private
 
+bool MetadataJob::fetchMetaDataPackages()
+{
+    //Download files in chunks. QtConcurrent will choke if too many task is given to it
+    int chunkSize = qMin(m_packages.length(), m_downloadableChunkSize);
+    QList<FileTaskItem> tempPackages = m_packages.mid(0, chunkSize);
+    m_packages = m_packages.mid(chunkSize, m_packages.length());
+    if (tempPackages.length() > 0) {
+        m_taskNumber++;
+        setProcessedAmount(0);
+        DownloadFileTask *const metadataTask = new DownloadFileTask(tempPackages);
+        metadataTask->setProxyFactory(m_core->proxyFactory());
+        m_metadataTask.setFuture(QtConcurrent::run(&DownloadFileTask::doTask, metadataTask));
+        setProgressTotalAmount(100);
+        QString metaInformation;
+        if (m_totalTaskCount > 1)
+            metaInformation = tr("Retrieving meta information from remote repository... %1/%2 ").arg(m_taskNumber).arg(m_totalTaskCount);
+        else
+            metaInformation = tr("Retrieving meta information from remote repository... ");
+        emit infoMessage(this, metaInformation);
+        return true;
+    }
+    return false;
+}
+
 void MetadataJob::reset()
 {
     m_packages.clear();
@@ -615,6 +638,9 @@ MetadataJob::Status MetadataJob::parseUpdatesXml(const QList<FileTaskResult> &re
             }
         }
     }
+    double taskCount = m_packages.length()/static_cast<double>(m_downloadableChunkSize);
+    m_totalTaskCount = ceil(taskCount);
+
     return XmlDownloadSuccess;
 }
 
diff --git a/src/libs/installer/metadatajob.h b/src/libs/installer/metadatajob.h
index c6d5ad353..f3f480f6c 100644
--- a/src/libs/installer/metadatajob.h
+++ b/src/libs/installer/metadatajob.h
@@ -79,6 +79,7 @@ private slots:
     void startXMLTask(const QList<FileTaskItem> items);
 
 private:
+    bool fetchMetaDataPackages();
     void startUnzipRepositoryTask(const Repository &repo);
     void reset();
     void resetCompressedFetch();
@@ -96,6 +97,10 @@ private:
     QHash<QFutureWatcher<void> *, QObject*> m_unzipRepositoryTasks;
     bool m_addCompressedPackages;
     QList<FileTaskItem> m_unzipRepositoryitems;
+    QList<FileTaskResult> m_metadataResult;
+    int m_downloadableChunkSize;
+    int m_taskNumber;
+    int m_totalTaskCount;
 };
 
 }   // namespace QInstaller