Add support for job pools

Commands can now be assigned to an arbitrary job pool and a limit for
the number of concurrently running jobs in such pools can be provided in
a number of ways:
- via the build command line: qbs --job-limits linker:1
- via the settings: qbs config preferences.jobLimit.linker 1
- in a project file: JobLimit {
                         jobPool: "linker";
                         jobCount: 1
                     }
We provide two job pools ourselves with the cpp module: "compiler" and
"linker".

[ChangeLog] Added the concept of job pools for limiting concurrent
execution of commands by type

Task-number: QBS-743
Change-Id: Ib3f361dbc73093e342bf0eba0daf2079a2b3a8ce
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@qt.io>
Reviewed-by: Joerg Bornemann <joerg.bornemann@qt.io>

24 files changed, 569 insertions, 9 deletions

diff --git a/src/lib/corelib/buildgraph/executor.cpp b/src/lib/corelib/buildgraph/executor.cpp
index f03529eec..368fa6ccb 100644
--- a/src/lib/corelib/buildgraph/executor.cpp
+++ b/src/lib/corelib/buildgraph/executor.cpp
@@ -61,10 +61,12 @@
 #include <logging/translator.h>
 #include <tools/error.h>
 #include <tools/fileinfo.h>
+#include <tools/preferences.h>
 #include <tools/profiling.h>
 #include <tools/progressobserver.h>
 #include <tools/qbsassert.h>
 #include <tools/qttools.h>
+#include <tools/settings.h>
 #include <tools/stringconstants.h>
 
 #include <QtCore/qdir.h>
@@ -230,6 +232,9 @@ void Executor::doBuild()
     m_tagsNeededForFilesToConsider.clear();
     m_productsOfFilesToConsider.clear();
     m_artifactsRemovedFromDisk.clear();
+    m_jobCountPerPool.clear();
+
+    setupJobLimits();
 
     // TODO: The "filesToConsider" thing is badly designed; we should know exactly which artifact
     //       it is. Remove this from the BuildOptions class and introduce Project::buildSomeFiles()
@@ -346,6 +351,7 @@ void Executor::updateLeaves(BuildGraphNode *node, NodeSet &seenNodes)
 bool Executor::scheduleJobs()
 {
     QBS_CHECK(m_state == ExecutorRunning);
+    std::vector<BuildGraphNode *> delayedLeaves;
     while (!m_leaves.empty() && !m_availableJobs.empty()) {
         BuildGraphNode * const nodeToBuild = m_leaves.top();
         m_leaves.pop();
@@ -355,9 +361,17 @@ bool Executor::scheduleJobs()
             QBS_ASSERT(!"untouched node in leaves list",
                        qDebug("%s", qPrintable(nodeToBuild->toString())));
             break;
-        case BuildGraphNode::Buildable:
-            // This is the only state in which we want to build a node.
-            nodeToBuild->accept(this);
+        case BuildGraphNode::Buildable: // This is the only state in which we want to build a node.
+            // TODO: It's a bit annoying that we have to check this here already, when we
+            //       don't know whether the transformer needs to run at all. Investigate
+            //       moving the whole job allocation logic to runTransformer().
+            if (schedulingBlockedByJobLimit(nodeToBuild)) {
+                qCDebug(lcExec).noquote() << "node delayed due to occupied job pool:"
+                                          << nodeToBuild->toString();
+                delayedLeaves.push_back(nodeToBuild);
+            } else {
+                nodeToBuild->accept(this);
+            }
             break;
         case BuildGraphNode::Building:
             qCDebug(lcExec).noquote() << nodeToBuild->toString();
@@ -369,9 +383,50 @@ bool Executor::scheduleJobs()
             break;
         }
     }
+    for (BuildGraphNode * const delayedLeaf : delayedLeaves)
+        m_leaves.push(delayedLeaf);
     return !m_leaves.empty() || !m_processingJobs.empty();
 }
 
+bool Executor::schedulingBlockedByJobLimit(const BuildGraphNode *node)
+{
+    if (node->type() != BuildGraphNode::ArtifactNodeType)
+        return false;
+    const Artifact * const artifact = static_cast<const Artifact *>(node);
+    if (artifact->artifactType == Artifact::SourceFile)
+        return false;
+
+    const Transformer * const transformer = artifact->transformer.get();
+    for (const QString &jobPool : transformer->jobPools()) {
+        const int currentJobCount = m_jobCountPerPool[jobPool];
+        if (currentJobCount == 0)
+            continue;
+        const auto jobLimitIsExceeded = [currentJobCount, jobPool, this](const Transformer *t) {
+            const int maxJobCount = m_jobLimitsPerProduct.at(t->product().get())
+                    .getLimit(jobPool);
+            return maxJobCount > 0 && currentJobCount >= maxJobCount;
+        };
+
+        // Different products can set different limits. The effective limit is the minimum of what
+        // is set in this transformer's product and in the products of all currently
+        // running transformers.
+        if (jobLimitIsExceeded(transformer))
+            return true;
+        for (const ExecutorJob * const runningJob : m_processingJobs.keys()) {
+            if (!runningJob->jobPools().contains(jobPool))
+                continue;
+            const Transformer * const runningTransformer = runningJob->transformer();
+            if (!runningTransformer)
+                continue; // This can happen if the ExecutorJob has already finished.
+            if (runningTransformer->product() == transformer->product())
+                continue; // We have already checked this product's job limit.
+            if (jobLimitIsExceeded(runningTransformer))
+                return true;
+        }
+    }
+    return false;
+}
+
 bool Executor::isUpToDate(Artifact *artifact) const
 {
     QBS_CHECK(artifact->artifactType == Artifact::Generated);
@@ -503,6 +558,7 @@ void Executor::finishJob(ExecutorJob *job, bool success)
     const TransformerPtr transformer = it.value();
     m_processingJobs.erase(it);
     m_availableJobs.push_back(job);
+    updateJobCounts(transformer.get(), -1);
     if (success) {
         m_project->buildData->setDirty();
         for (Artifact * const artifact : qAsConst(transformer->outputs)) {
@@ -630,6 +686,30 @@ bool Executor::transformerHasMatchingInputFiles(const TransformerConstPtr &trans
     return false;
 }
 
+void Executor::setupJobLimits()
+{
+    Settings settings(m_buildOptions.settingsDirectory());
+    for (const ResolvedProductConstPtr &p : m_productsToBuild) {
+        const Preferences prefs(&settings, p->profile());
+        const JobLimits &jobLimitsFromSettings = prefs.jobLimits();
+        JobLimits effectiveJobLimits;
+        if (m_buildOptions.projectJobLimitsTakePrecedence()) {
+            effectiveJobLimits.update(jobLimitsFromSettings).update(m_buildOptions.jobLimits())
+                    .update(p->jobLimits);
+        } else {
+            effectiveJobLimits.update(p->jobLimits).update(jobLimitsFromSettings)
+                    .update(m_buildOptions.jobLimits());
+        }
+        m_jobLimitsPerProduct.insert(std::make_pair(p.get(), effectiveJobLimits));
+    }
+}
+
+void Executor::updateJobCounts(const Transformer *transformer, int diff)
+{
+    for (const QString &jobPool : transformer->jobPools())
+        m_jobCountPerPool[jobPool] += diff;
+}
+
 void Executor::cancelJobs()
 {
     if (m_state == ExecutorCanceling)
@@ -923,6 +1003,7 @@ void Executor::runTransformer(const TransformerPtr &transformer)
     for (Artifact * const artifact : qAsConst(transformer->outputs))
         artifact->buildState = BuildGraphNode::Building;
     m_processingJobs.insert(job, transformer);
+    updateJobCounts(transformer.get(), 1);
     job->run(transformer.get());
 }
 
diff --git a/src/lib/corelib/buildgraph/executor.h b/src/lib/corelib/buildgraph/executor.h
index 7225c0ace..4b4951e1e 100644
--- a/src/lib/corelib/buildgraph/executor.h
+++ b/src/lib/corelib/buildgraph/executor.h
@@ -154,6 +154,10 @@ private:
     bool artifactHasMatchingOutputTags(const Artifact *artifact) const;
     bool transformerHasMatchingInputFiles(const TransformerConstPtr &transformer) const;
 
+    void setupJobLimits();
+    void updateJobCounts(const Transformer *transformer, int diff);
+    bool schedulingBlockedByJobLimit(const BuildGraphNode *node);
+
     typedef QHash<ExecutorJob *, TransformerPtr> JobMap;
     JobMap m_processingJobs;
 
@@ -169,6 +173,8 @@ private:
     std::vector<ResolvedProductPtr> m_allProducts;
     std::unordered_map<QString, const ResolvedProduct *> m_productsByName;
     std::unordered_map<QString, const ResolvedProject *> m_projectsByName;
+    std::unordered_map<QString, int> m_jobCountPerPool;
+    std::unordered_map<const ResolvedProduct *, JobLimits> m_jobLimitsPerProduct;
     NodeSet m_roots;
     Leaves m_leaves;
     InputArtifactScannerContext *m_inputArtifactScanContext;
diff --git a/src/lib/corelib/buildgraph/executorjob.cpp b/src/lib/corelib/buildgraph/executorjob.cpp
index e90b498f6..79f17377d 100644
--- a/src/lib/corelib/buildgraph/executorjob.cpp
+++ b/src/lib/corelib/buildgraph/executorjob.cpp
@@ -113,6 +113,7 @@ void ExecutorJob::run(Transformer *t)
     m_processCommandExecutor->setProcessEnvironment(
                 (*t->outputs.cbegin())->product->buildEnvironment);
     m_transformer = t;
+    m_jobPools = t->jobPools();
     runNextCommand();
 }
 
@@ -171,6 +172,7 @@ void ExecutorJob::setFinished()
 void ExecutorJob::reset()
 {
     m_transformer = nullptr;
+    m_jobPools.clear();
     m_currentCommandExecutor = nullptr;
     m_currentCommandIdx = -1;
     m_error.clear();
diff --git a/src/lib/corelib/buildgraph/executorjob.h b/src/lib/corelib/buildgraph/executorjob.h
index b1701a11d..e28d42f7f 100644
--- a/src/lib/corelib/buildgraph/executorjob.h
+++ b/src/lib/corelib/buildgraph/executorjob.h
@@ -43,8 +43,10 @@
 #include <language/forward_decls.h>
 #include <tools/commandechomode.h>
 #include <tools/error.h>
+#include <tools/set.h>
 
 #include <QtCore/qobject.h>
+#include <QtCore/qstring.h>
 
 namespace qbs {
 class CodeLocation;
@@ -71,6 +73,8 @@ public:
     void setEchoMode(CommandEchoMode echoMode);
     void run(Transformer *t);
     void cancel();
+    const Transformer *transformer() const { return m_transformer; }
+    Set<QString> jobPools() const { return m_jobPools; }
 
 signals:
     void reportCommandDescription(const QString &highlight, const QString &message);
@@ -88,6 +92,7 @@ private:
     ProcessCommandExecutor *m_processCommandExecutor;
     JsCommandExecutor *m_jsCommandExecutor;
     Transformer *m_transformer;
+    Set<QString> m_jobPools;
     int m_currentCommandIdx;
     ErrorInfo m_error;
 };
diff --git a/src/lib/corelib/buildgraph/rulecommands.cpp b/src/lib/corelib/buildgraph/rulecommands.cpp
index c2aad1ce1..f7620dbf8 100644
--- a/src/lib/corelib/buildgraph/rulecommands.cpp
+++ b/src/lib/corelib/buildgraph/rulecommands.cpp
@@ -103,6 +103,7 @@ bool AbstractCommand::equals(const AbstractCommand *other) const
             && m_highlight == other->m_highlight
             && m_ignoreDryRun == other->m_ignoreDryRun
             && m_silent == other->m_silent
+            && m_jobPool == other->m_jobPool
             && m_properties == other->m_properties;
 }
 
@@ -113,6 +114,7 @@ void AbstractCommand::fillFromScriptValue(const QScriptValue *scriptValue, const
     m_highlight = scriptValue->property(highlightProperty()).toString();
     m_ignoreDryRun = scriptValue->property(ignoreDryRunProperty()).toBool();
     m_silent = scriptValue->property(silentProperty()).toBool();
+    m_jobPool = scriptValue->property(StringConstants::jobPoolProperty()).toString();
     m_codeLocation = codeLocation;
 
     m_predefinedProperties
@@ -120,6 +122,7 @@ void AbstractCommand::fillFromScriptValue(const QScriptValue *scriptValue, const
             << extendedDescriptionProperty()
             << highlightProperty()
             << ignoreDryRunProperty()
+            << StringConstants::jobPoolProperty()
             << silentProperty();
 }
 
diff --git a/src/lib/corelib/buildgraph/rulecommands.h b/src/lib/corelib/buildgraph/rulecommands.h
index 7583de2b6..02d1a02a5 100644
--- a/src/lib/corelib/buildgraph/rulecommands.h
+++ b/src/lib/corelib/buildgraph/rulecommands.h
@@ -81,6 +81,7 @@ public:
     const QString highlight() const { return m_highlight; }
     bool ignoreDryRun() const { return m_ignoreDryRun; }
     bool isSilent() const { return m_silent; }
+    QString jobPool() const { return m_jobPool; }
     CodeLocation codeLocation() const { return m_codeLocation; }
 
     const QVariantMap &properties() const { return m_properties; }
@@ -98,7 +99,8 @@ private:
     template<PersistentPool::OpType opType> void serializationOp(PersistentPool &pool)
     {
         pool.serializationOp<opType>(m_description, m_extendedDescription, m_highlight,
-                                     m_ignoreDryRun, m_silent, m_codeLocation, m_properties);
+                                     m_ignoreDryRun, m_silent, m_codeLocation, m_jobPool,
+                                     m_properties);
     }
 
     QString m_description;
@@ -107,6 +109,7 @@ private:
     bool m_ignoreDryRun;
     bool m_silent;
     CodeLocation m_codeLocation;
+    QString m_jobPool;
     QVariantMap m_properties;
 };
 
diff --git a/src/lib/corelib/buildgraph/transformer.cpp b/src/lib/corelib/buildgraph/transformer.cpp
index df371e1fe..4569b9955 100644
--- a/src/lib/corelib/buildgraph/transformer.cpp
+++ b/src/lib/corelib/buildgraph/transformer.cpp
@@ -305,5 +305,15 @@ void Transformer::rescueChangeTrackingData(const TransformerConstPtr &other)
     exportedModulesAccessedInCommands = other->exportedModulesAccessedInCommands;
 }
 
+Set<QString> Transformer::jobPools() const
+{
+    Set<QString> pools;
+    for (const AbstractCommandPtr &c : commands.commands()) {
+        if (!c->jobPool().isEmpty())
+            pools.insert(c->jobPool());
+    }
+    return pools;
+}
+
 } // namespace Internal
 } // namespace qbs
diff --git a/src/lib/corelib/buildgraph/transformer.h b/src/lib/corelib/buildgraph/transformer.h
index ce66a976e..2f6a8e56d 100644
--- a/src/lib/corelib/buildgraph/transformer.h
+++ b/src/lib/corelib/buildgraph/transformer.h
@@ -102,6 +102,8 @@ public:
                         const QScriptValueList &args);
     void rescueChangeTrackingData(const TransformerConstPtr &other);
 
+    Set<QString> jobPools() const;
+
     template<PersistentPool::OpType opType> void completeSerializationOp(PersistentPool &pool)
     {
         pool.serializationOp<opType>(rule, inputs, outputs, explicitlyDependsOn,
diff --git a/src/lib/corelib/corelib.qbs b/src/lib/corelib/corelib.qbs
index 93509763b..c947cb484 100644
--- a/src/lib/corelib/corelib.qbs
+++ b/src/lib/corelib/corelib.qbs
@@ -397,6 +397,7 @@ QbsLibrary {
             "id.cpp",
             "id.h",
             "iosutils.h",
+            "joblimits.cpp",
             "jsliterals.cpp",
             "jsliterals.h",
             "installoptions.cpp",
@@ -467,6 +468,7 @@ QbsLibrary {
             "error.h",
             "generateoptions.h",
             "installoptions.h",
+            "joblimits.h",
             "preferences.h",
             "processresult.h",
             "profile.h",
diff --git a/src/lib/corelib/language/builtindeclarations.cpp b/src/lib/corelib/language/builtindeclarations.cpp
index 4be405112..4886675d0 100644
--- a/src/lib/corelib/language/builtindeclarations.cpp
+++ b/src/lib/corelib/language/builtindeclarations.cpp
@@ -67,6 +67,7 @@ BuiltinDeclarations::BuiltinDeclarations()
         { QLatin1String("Export"), ItemType::Export },
         { QLatin1String("FileTagger"), ItemType::FileTagger },
         { QLatin1String("Group"), ItemType::Group },
+        { QLatin1String("JobLimit"), ItemType::JobLimit },
         { QLatin1String("Module"), ItemType::Module },
         { QLatin1String("Parameter"), ItemType::Parameter },
         { QLatin1String("Parameters"), ItemType::Parameters },
@@ -87,6 +88,7 @@ BuiltinDeclarations::BuiltinDeclarations()
     addExportItem();
     addFileTaggerItem();
     addGroupItem();
+    addJobLimitItem();
     addModuleItem();
     addProbeItem();
     addProductItem();
@@ -298,6 +300,15 @@ void BuiltinDeclarations::addGroupItem()
     insert(item);
 }
 
+void BuiltinDeclarations::addJobLimitItem()
+{
+    ItemDeclaration item(ItemType::JobLimit);
+    item << conditionProperty();
+    item << PropertyDeclaration(StringConstants::jobPoolProperty(), PropertyDeclaration::String);
+    item << PropertyDeclaration(StringConstants::jobCountProperty(), PropertyDeclaration::Integer);
+    insert(item);
+}
+
 void BuiltinDeclarations::addModuleItem()
 {
     ItemDeclaration item = moduleLikeItem(ItemType::Module);
@@ -312,6 +323,7 @@ ItemDeclaration BuiltinDeclarations::moduleLikeItem(ItemType type)
             << ItemType::Group
             << ItemType::Depends
             << ItemType::FileTagger
+            << ItemType::JobLimit
             << ItemType::Rule
             << ItemType::Parameter
             << ItemType::Probe
@@ -360,6 +372,7 @@ void BuiltinDeclarations::addProductItem()
             << ItemType::Depends
             << ItemType::Group
             << ItemType::FileTagger
+            << ItemType::JobLimit
             << ItemType::Export
             << ItemType::Probe
             << ItemType::Profile
@@ -429,6 +442,7 @@ void BuiltinDeclarations::addProjectItem()
             << ItemType::Profile
             << ItemType::Probe
             << ItemType::FileTagger
+            << ItemType::JobLimit
             << ItemType::Rule);
     item << nameProperty();
     item << conditionProperty();
diff --git a/src/lib/corelib/language/builtindeclarations.h b/src/lib/corelib/language/builtindeclarations.h
index c75475df7..ff16b395a 100644
--- a/src/lib/corelib/language/builtindeclarations.h
+++ b/src/lib/corelib/language/builtindeclarations.h
@@ -76,6 +76,7 @@ private:
     void addExportItem();
     void addFileTaggerItem();
     void addGroupItem();
+    void addJobLimitItem();
     void addModuleItem();
     static ItemDeclaration moduleLikeItem(ItemType type);
     void addProbeItem();
diff --git a/src/lib/corelib/language/itemtype.h b/src/lib/corelib/language/itemtype.h
index c0e76c94b..324a1fb87 100644
--- a/src/lib/corelib/language/itemtype.h
+++ b/src/lib/corelib/language/itemtype.h
@@ -53,6 +53,7 @@ enum class ItemType {
     Export,
     FileTagger,
     Group,
+    JobLimit,
     Module,
     Parameter,
     Parameters,
diff --git a/src/lib/corelib/language/language.h b/src/lib/corelib/language/language.h
index 580b3810c..06a9af54a 100644
--- a/src/lib/corelib/language/language.h
+++ b/src/lib/corelib/language/language.h
@@ -49,6 +49,7 @@
 #include <buildgraph/forward_decls.h>
 #include <tools/codelocation.h>
 #include <tools/filetime.h>
+#include <tools/joblimits.h>
 #include <tools/persistence.h>
 #include <tools/set.h>
 #include <tools/weakpointer.h>
@@ -571,6 +572,7 @@ public:
     std::vector<ResolvedProductPtr> dependencies;
     QHash<ResolvedProductConstPtr, QVariantMap> dependencyParameters;
     std::vector<FileTaggerConstPtr> fileTaggers;
+    JobLimits jobLimits;
     std::vector<ResolvedModulePtr> modules;
     QHash<ResolvedModuleConstPtr, QVariantMap> moduleParameters;
     std::vector<ResolvedScannerConstPtr> scanners;
@@ -628,7 +630,8 @@ private:
                                      missingSourceFiles, location, productProperties,
                                      moduleProperties, rules, dependencies, dependencyParameters,
                                      fileTaggers, modules, moduleParameters, scanners, groups,
-                                     artifactProperties, probes, exportedModule, buildData);
+                                     artifactProperties, probes, exportedModule, buildData,
+                                     jobLimits);
     }
 
     QHash<QString, QString> m_executablePathCache;
diff --git a/src/lib/corelib/language/projectresolver.cpp b/src/lib/corelib/language/projectresolver.cpp
index ec29e00c0..27829e378 100644
--- a/src/lib/corelib/language/projectresolver.cpp
+++ b/src/lib/corelib/language/projectresolver.cpp
@@ -56,6 +56,7 @@
 #include <logging/translator.h>
 #include <tools/error.h>
 #include <tools/fileinfo.h>
+#include <tools/joblimits.h>
 #include <tools/jsliterals.h>
 #include <tools/profiling.h>
 #include <tools/progressobserver.h>
@@ -89,6 +90,7 @@ struct ProjectResolver::ProjectContext
     ResolvedProjectPtr project;
     std::vector<FileTaggerConstPtr> fileTaggers;
     std::vector<RulePtr> rules;
+    JobLimits jobLimits;
     ResolvedModulePtr dummyModule;
 };
 
@@ -106,6 +108,7 @@ struct ProjectResolver::ProductContext
 struct ProjectResolver::ModuleContext
 {
     ResolvedModulePtr module;
+    JobLimits jobLimits;
 };
 
 class CancelException { };
@@ -333,6 +336,7 @@ void ProjectResolver::resolveProjectFully(Item *item, ProjectResolver::ProjectCo
         { ItemType::Product, &ProjectResolver::resolveProduct },
         { ItemType::Probe, &ProjectResolver::ignoreItem },
         { ItemType::FileTagger, &ProjectResolver::resolveFileTagger },
+        { ItemType::JobLimit, &ProjectResolver::resolveJobLimit },
         { ItemType::Rule, &ProjectResolver::resolveRule },
         { ItemType::PropertyOptions, &ProjectResolver::ignoreItem }
     };
@@ -494,6 +498,7 @@ void ProjectResolver::resolveProductFully(Item *item, ProjectContext *projectCon
         { ItemType::Depends, &ProjectResolver::ignoreItem },
         { ItemType::Rule, &ProjectResolver::resolveRule },
         { ItemType::FileTagger, &ProjectResolver::resolveFileTagger },
+        { ItemType::JobLimit, &ProjectResolver::resolveJobLimit },
         { ItemType::Group, &ProjectResolver::resolveGroup },
         { ItemType::Product, &ProjectResolver::resolveShadowProduct },
         { ItemType::Export, &ProjectResolver::resolveExport },
@@ -504,6 +509,11 @@ void ProjectResolver::resolveProductFully(Item *item, ProjectContext *projectCon
     for (Item * const child : qAsConst(subItems))
         callItemFunction(mapping, child, projectContext);
 
+    for (const ProjectContext *p = projectContext; p; p = p->parentContext) {
+        JobLimits tempLimits = p->jobLimits;
+        product->jobLimits = tempLimits.update(product->jobLimits);
+    }
+
     resolveModules(item, projectContext);
 
     for (const FileTag &t : qAsConst(product->fileTags))
@@ -512,12 +522,19 @@ void ProjectResolver::resolveProductFully(Item *item, ProjectContext *projectCon
 
 void ProjectResolver::resolveModules(const Item *item, ProjectContext *projectContext)
 {
+    JobLimits jobLimits;
     for (const Item::Module &m : item->modules())
-        resolveModule(m.name, m.item, m.isProduct, m.parameters, projectContext);
+        resolveModule(m.name, m.item, m.isProduct, m.parameters, jobLimits, projectContext);
+    for (int i = 0; i < jobLimits.count(); ++i) {
+        const JobLimit &moduleJobLimit = jobLimits.jobLimitAt(i);
+        if (m_productContext->product->jobLimits.getLimit(moduleJobLimit.pool()) == -1)
+            m_productContext->product->jobLimits.setJobLimit(moduleJobLimit);
+    }
 }
 
 void ProjectResolver::resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct,
-                                    const QVariantMap &parameters, ProjectContext *projectContext)
+                                    const QVariantMap &parameters, JobLimits &jobLimits,
+                                    ProjectContext *projectContext)
 {
     checkCancelation();
     if (!item->isPresentModule())
@@ -550,6 +567,7 @@ void ProjectResolver::resolveModule(const QualifiedId &moduleName, Item *item, b
         { ItemType::Group, &ProjectResolver::ignoreItem },
         { ItemType::Rule, &ProjectResolver::resolveRule },
         { ItemType::FileTagger, &ProjectResolver::resolveFileTagger },
+        { ItemType::JobLimit, &ProjectResolver::resolveJobLimit },
         { ItemType::Scanner, &ProjectResolver::resolveScanner },
         { ItemType::PropertyOptions, &ProjectResolver::ignoreItem },
         { ItemType::Depends, &ProjectResolver::ignoreItem },
@@ -559,6 +577,12 @@ void ProjectResolver::resolveModule(const QualifiedId &moduleName, Item *item, b
     };
     for (Item *child : item->children())
         callItemFunction(mapping, child, projectContext);
+    for (int i = 0; i < moduleContext.jobLimits.count(); ++i) {
+        const JobLimit &newJobLimit = moduleContext.jobLimits.jobLimitAt(i);
+        const int oldLimit = jobLimits.getLimit(newJobLimit.pool());
+        if (oldLimit == -1 || oldLimit > newJobLimit.limit())
+            jobLimits.setJobLimit(newJobLimit);
+    }
 
     m_moduleContext = oldModuleContext;
 }
@@ -1314,6 +1338,35 @@ void ProjectResolver::resolveFileTagger(Item *item, ProjectContext *projectConte
     fileTaggers.push_back(FileTagger::create(patterns, fileTags, priority));
 }
 
+void ProjectResolver::resolveJobLimit(Item *item, ProjectResolver::ProjectContext *projectContext)
+{
+    if (!m_evaluator->boolValue(item, StringConstants::conditionProperty()))
+        return;
+    const QString jobPool = m_evaluator->stringValue(item, StringConstants::jobPoolProperty());
+    if (jobPool.isEmpty())
+        throw ErrorInfo(Tr::tr("A JobLimit item needs to have a non-empty '%1' property.")
+                        .arg(StringConstants::jobPoolProperty()), item->location());
+    bool jobCountWasSet;
+    const int jobCount = m_evaluator->intValue(item, StringConstants::jobCountProperty(), -1,
+                                               &jobCountWasSet);
+    if (!jobCountWasSet) {
+        throw ErrorInfo(Tr::tr("A JobLimit item needs to have a '%1' property.")
+                        .arg(StringConstants::jobCountProperty()), item->location());
+    }
+    if (jobCount < 0) {
+        throw ErrorInfo(Tr::tr("A JobLimit item must have a non-negative '%1' property.")
+                        .arg(StringConstants::jobCountProperty()), item->location());
+    }
+    JobLimits &jobLimits = m_moduleContext
+            ? m_moduleContext->jobLimits
+            : m_productContext ? m_productContext->product->jobLimits
+            : projectContext->jobLimits;
+    JobLimit jobLimit(jobPool, jobCount);
+    const int oldLimit = jobLimits.getLimit(jobPool);
+    if (oldLimit == -1 || oldLimit > jobCount)
+        jobLimits.setJobLimit(jobLimit);
+}
+
 void ProjectResolver::resolveScanner(Item *item, ProjectResolver::ProjectContext *projectContext)
 {
     checkCancelation();
diff --git a/src/lib/corelib/language/projectresolver.h b/src/lib/corelib/language/projectresolver.h
index 660bf6259..63a083fef 100644
--- a/src/lib/corelib/language/projectresolver.h
+++ b/src/lib/corelib/language/projectresolver.h
@@ -56,6 +56,7 @@
 #include <vector>
 
 namespace qbs {
+class JobLimits;
 namespace Internal {
 
 class Evaluator;
@@ -101,7 +102,8 @@ private:
     void resolveProductFully(Item *item, ProjectContext *projectContext);
     void resolveModules(const Item *item, ProjectContext *projectContext);
     void resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct,
-                       const QVariantMap &parameters, ProjectContext *projectContext);
+                       const QVariantMap &parameters, JobLimits &jobLimits,
+                       ProjectContext *projectContext);
     void gatherProductTypes(ResolvedProduct *product, Item *item);
     QVariantMap resolveAdditionalModuleProperties(const Item *group,
                                                   const QVariantMap &currentValues);
@@ -117,6 +119,7 @@ private:
                                     const QStringList &namePrefix,
                                     QualifiedIdSet *seenBindings);
     void resolveFileTagger(Item *item, ProjectContext *projectContext);
+    void resolveJobLimit(Item *item, ProjectContext *projectContext);
     void resolveScanner(Item *item, ProjectContext *projectContext);
     void resolveProductDependencies(const ProjectContext &projectContext);
     void postProcess(const ResolvedProductPtr &product, ProjectContext *projectContext) const;
diff --git a/src/lib/corelib/tools/buildoptions.cpp b/src/lib/corelib/tools/buildoptions.cpp
index ac8e39867..5507e0842 100644
--- a/src/lib/corelib/tools/buildoptions.cpp
+++ b/src/lib/corelib/tools/buildoptions.cpp
@@ -58,6 +58,8 @@ public:
     QStringList changedFiles;
     QStringList filesToConsider;
     QStringList activeFileTags;
+    JobLimits jobLimits;
+    QString settingsDir;
     int maxJobCount;
     bool dryRun;
     bool keepGoing;
@@ -68,6 +70,7 @@ public:
     bool install;
     bool removeExistingInstallation;
     bool onlyExecuteRules;
+    bool jobLimitsFromProjectTakePrecedence = false;
 };
 
 } // namespace Internal
@@ -189,6 +192,44 @@ void BuildOptions::setMaxJobCount(int jobCount)
 }
 
 /*!
+ * \brief The base directory for qbs settings.
+ * This value is used to locate profiles and preferences.
+ */
+QString BuildOptions::settingsDirectory() const
+{
+    return d->settingsDir;
+}
+
+/*!
+ * \brief Sets the base directory for qbs settings.
+ * \param settingsBaseDir Will be used to locate profiles and preferences.
+ */
+void BuildOptions::setSettingsDirectory(const QString &settingsBaseDir)
+{
+    d->settingsDir = settingsBaseDir;
+}
+
+JobLimits BuildOptions::jobLimits() const
+{
+    return d->jobLimits;
+}
+
+void BuildOptions::setJobLimits(const JobLimits &jobLimits)
+{
+    d->jobLimits = jobLimits;
+}
+
+bool BuildOptions::projectJobLimitsTakePrecedence() const
+{
+    return d->jobLimitsFromProjectTakePrecedence;
+}
+
+void BuildOptions::setProjectJobLimitsTakePrecedence(bool toggle)
+{
+    d->jobLimitsFromProjectTakePrecedence = toggle;
+}
+
+/*!
  * \brief Returns true iff qbs will not actually execute any commands, but just show what
  *        would happen.
  * The default is false.
diff --git a/src/lib/corelib/tools/buildoptions.h b/src/lib/corelib/tools/buildoptions.h
index 630a6aa22..cea89d0ea 100644
--- a/src/lib/corelib/tools/buildoptions.h
+++ b/src/lib/corelib/tools/buildoptions.h
@@ -42,6 +42,7 @@
 #include "qbs_export.h"
 
 #include "commandechomode.h"
+#include "joblimits.h"
 
 #include <QtCore/qshareddata.h>
 
@@ -73,6 +74,15 @@ public:
     int maxJobCount() const;
     void setMaxJobCount(int jobCount);
 
+    QString settingsDirectory() const;
+    void setSettingsDirectory(const QString &settingsBaseDir);
+
+    JobLimits jobLimits() const;
+    void setJobLimits(const JobLimits &jobLimits);
+
+    bool projectJobLimitsTakePrecedence() const;
+    void setProjectJobLimitsTakePrecedence(bool toggle);
+
     bool dryRun() const;
     void setDryRun(bool dryRun);
 
diff --git a/src/lib/corelib/tools/joblimits.cpp b/src/lib/corelib/tools/joblimits.cpp
new file mode 100644
index 000000000..3b1fde83d
--- /dev/null
+++ b/src/lib/corelib/tools/joblimits.cpp
@@ -0,0 +1,185 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qbs.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "joblimits.h"
+
+#include <tools/persistence.h>
+
+#include <utility>
+#include <vector>
+
+namespace qbs {
+namespace Internal {
+
+static int transformLimit(int limitFromUser)
+{
+    return limitFromUser == 0
+            ? std::numeric_limits<int>::max()
+            : limitFromUser < -1 ? -1
+            : limitFromUser;
+}
+
+class JobLimitPrivate : public QSharedData
+{
+public:
+    JobLimitPrivate(const QString &pool, int limit)
+        : jobLimit(std::make_pair(pool, transformLimit(limit)))
+    {
+    }
+    template<PersistentPool::OpType opType> void serializationOp(PersistentPool &pool)
+    {
+        pool.serializationOp<opType>(jobLimit);
+    }
+    std::pair<QString, int> jobLimit;
+};
+
+class JobLimitsPrivate : public QSharedData
+{
+public:
+    template<PersistentPool::OpType opType> void serializationOp(PersistentPool &pool)
+    {
+        pool.serializationOp<opType>(jobLimits);
+    }
+    std::vector<JobLimit> jobLimits;
+};
+
+} // namespace Internal
+
+JobLimit::JobLimit() : JobLimit(QString(), -1)
+{
+}
+JobLimit::JobLimit(const QString &pool, int limit) : d(new Internal::JobLimitPrivate(pool, limit))
+{
+}
+JobLimit::JobLimit(const JobLimit &other) : d(other.d) { }
+JobLimit &JobLimit::operator=(const JobLimit &other)
+{
+    d = other.d;
+    return *this;
+}
+JobLimit::~JobLimit() {}
+QString JobLimit::pool() const { return d->jobLimit.first; }
+int JobLimit::limit() const { return d->jobLimit.second; }
+
+void JobLimit::load(Internal::PersistentPool &pool)
+{
+    d->serializationOp<Internal::PersistentPool::Load>(pool);
+}
+
+void JobLimit::store(Internal::PersistentPool &pool)
+{
+    d->serializationOp<Internal::PersistentPool::Store>(pool);
+}
+
+JobLimits::JobLimits() : d(new Internal::JobLimitsPrivate) { }
+JobLimits::JobLimits(const JobLimits &other) : d(other.d) { }
+JobLimits &JobLimits::operator=(const JobLimits &other)
+{
+    d = other.d;
+    return *this;
+}
+JobLimits::~JobLimits() {}
+
+void JobLimits::setJobLimit(const JobLimit &limit)
+{
+    for (std::size_t i = 0; i < d->jobLimits.size(); ++i) {
+        JobLimit &currentLimit = d->jobLimits.at(i);
+        if (currentLimit.pool() == limit.pool()) {
+            if (currentLimit.limit() != limit.limit())
+                currentLimit = limit;
+            return;
+        }
+    }
+    d->jobLimits.push_back(limit);
+}
+
+void JobLimits::setJobLimit(const QString &pool, int limit)
+{
+    setJobLimit(JobLimit(pool, limit));
+}
+
+int JobLimits::getLimit(const QString &pool) const
+{
+    for (const JobLimit &l : d->jobLimits) {
+        if (l.pool() == pool)
+            return l.limit();
+    }
+    return -1;
+}
+
+bool JobLimits::isEmpty() const
+{
+    return d->jobLimits.empty();
+}
+
+int JobLimits::count() const
+{
+    return d->jobLimits.size();
+}
+
+JobLimit JobLimits::jobLimitAt(int i) const
+{
+    return d->jobLimits.at(i);
+}
+
+JobLimits &JobLimits::update(const JobLimits &other)
+{
+    if (isEmpty()) {
+        *this = other;
+    } else {
+        for (int i = 0; i < other.count(); ++i) {
+            const JobLimit &l = other.jobLimitAt(i);
+            if (l.limit() != -1)
+                setJobLimit(l);
+        }
+    }
+    return *this;
+}
+
+void JobLimits::load(Internal::PersistentPool &pool)
+{
+    d->serializationOp<Internal::PersistentPool::Load>(pool);
+}
+
+void JobLimits::store(Internal::PersistentPool &pool)
+{
+    d->serializationOp<Internal::PersistentPool::Store>(pool);
+}
+
+} // namespace qbs
diff --git a/src/lib/corelib/tools/joblimits.h b/src/lib/corelib/tools/joblimits.h
new file mode 100644
index 000000000..de95f5513
--- /dev/null
+++ b/src/lib/corelib/tools/joblimits.h
@@ -0,0 +1,102 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qbs.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+#ifndef QBS_JOB_LIMITS_H
+#define QBS_JOB_LIMITS_H
+
+#include "qbs_export.h"
+
+#include <QtCore/qshareddata.h>
+
+QT_BEGIN_NAMESPACE
+class QString;
+QT_END_NAMESPACE
+
+namespace qbs {
+namespace Internal {
+class JobLimitPrivate;
+class JobLimitsPrivate;
+class PersistentPool;
+}
+
+class QBS_EXPORT JobLimit
+{
+public:
+    JobLimit();
+    JobLimit(const QString &pool, int limit);
+    JobLimit(const JobLimit &other);
+    JobLimit &operator=(const JobLimit &other);
+    ~JobLimit();
+
+    QString pool() const;
+    int limit() const;
+
+    void load(Internal::PersistentPool &pool);
+    void store(Internal::PersistentPool &pool);
+private:
+    QSharedDataPointer<Internal::JobLimitPrivate> d;
+};
+
+class QBS_EXPORT JobLimits
+{
+public:
+    JobLimits();
+    JobLimits(const JobLimits &other);
+    JobLimits &operator=(const JobLimits &other);
+    ~JobLimits();
+
+    void setJobLimit(const JobLimit &limit);
+    void setJobLimit(const QString &pool, int limit);
+    int getLimit(const QString &pool) const;
+    bool hasLimit(const QString &pool) const { return getLimit(pool) != -1; }
+    bool isEmpty() const;
+
+    int count() const;
+    JobLimit jobLimitAt(int i) const;
+
+    JobLimits &update(const JobLimits &other);
+
+    void load(Internal::PersistentPool &pool);
+    void store(Internal::PersistentPool &pool);
+private:
+    QSharedDataPointer<Internal::JobLimitsPrivate> d;
+};
+
+} // namespace qbs
+
+#endif // include guard
diff --git a/src/lib/corelib/tools/persistence.cpp b/src/lib/corelib/tools/persistence.cpp
index 53c9dfac4..216dfad24 100644
--- a/src/lib/corelib/tools/persistence.cpp
+++ b/src/lib/corelib/tools/persistence.cpp
@@ -48,7 +48,7 @@
 namespace qbs {
 namespace Internal {
 
-static const char QBS_PERSISTENCE_MAGIC[] = "QBSPERSISTENCE-122";
+static const char QBS_PERSISTENCE_MAGIC[] = "QBSPERSISTENCE-123";
 
 NoBuildGraphError::NoBuildGraphError(const QString &filePath)
     : ErrorInfo(Tr::tr("Build graph not found for configuration '%1'. Expected location was '%2'.")
diff --git a/src/lib/corelib/tools/preferences.cpp b/src/lib/corelib/tools/preferences.cpp
index 66803a0f5..12af4e9c7 100644
--- a/src/lib/corelib/tools/preferences.cpp
+++ b/src/lib/corelib/tools/preferences.cpp
@@ -124,6 +124,32 @@ QStringList Preferences::pluginPaths(const QString &baseDir) const
     return pathList(QLatin1String("pluginsPath"), baseDir + QLatin1String("/qbs/plugins"));
 }
 
+/*!
+ * \brief Returns the per-pool job limits.
+ */
+JobLimits Preferences::jobLimits() const
+{
+    const QString prefix = QLatin1String("preferences.jobLimit");
+    JobLimits limits;
+    for (const QString &key : m_settings->allKeysWithPrefix(prefix, Settings::allScopes())) {
+        limits.setJobLimit(key, m_settings->value(prefix + QLatin1Char('.') + key,
+                                                  Settings::allScopes()).toInt());
+    }
+    const QString fullPrefix = prefix + QLatin1Char('.');
+    if (!m_profile.isEmpty()) {
+        Profile p(m_profile, m_settings, m_profileContents);
+        for (const QString &key : p.allKeys(Profile::KeySelectionRecursive)) {
+            if (!key.startsWith(fullPrefix))
+                continue;
+            const QString jobPool = key.mid(fullPrefix.size());
+            const int limit = p.value(key).toInt();
+            if (limit >= 0)
+                limits.setJobLimit(jobPool, limit);
+        }
+    }
+    return limits;
+}
+
 QVariant Preferences::getPreference(const QString &key, const QVariant &defaultValue) const
 {
     static const QString keyPrefix = QLatin1String("preferences");
diff --git a/src/lib/corelib/tools/preferences.h b/src/lib/corelib/tools/preferences.h
index 07f0edcd7..661b39d7f 100644
--- a/src/lib/corelib/tools/preferences.h
+++ b/src/lib/corelib/tools/preferences.h
@@ -42,6 +42,7 @@
 #include "qbs_export.h"
 
 #include "commandechomode.h"
+#include "joblimits.h"
 #include "settings.h"
 
 #include <QtCore/qstringlist.h>
@@ -63,6 +64,7 @@ public:
     CommandEchoMode defaultEchoMode() const;
     QStringList searchPaths(const QString &baseDir = QString()) const;
     QStringList pluginPaths(const QString &baseDir = QString()) const;
+    JobLimits jobLimits() const;
 
 private:
     QVariant getPreference(const QString &key, const QVariant &defaultValue = QVariant()) const;
diff --git a/src/lib/corelib/tools/stringconstants.h b/src/lib/corelib/tools/stringconstants.h
index f2666e070..6fcf3002b 100644
--- a/src/lib/corelib/tools/stringconstants.h
+++ b/src/lib/corelib/tools/stringconstants.h
@@ -104,6 +104,8 @@ public:
     QBS_STRING_CONSTANT(installPrefixProperty, "installPrefix")
     QBS_STRING_CONSTANT(installDirProperty, "installDir")
     QBS_STRING_CONSTANT(installSourceBaseProperty, "installSourceBase")
+    QBS_STRING_CONSTANT(jobCountProperty, "jobCount")
+    QBS_STRING_CONSTANT(jobPoolProperty, "jobPool")
     QBS_STRING_CONSTANT(lengthProperty, "length")
     QBS_STRING_CONSTANT(limitToSubProjectProperty, "limitToSubProject")
     QBS_STRING_CONSTANT(minimumQbsVersionProperty, "minimumQbsVersion")
diff --git a/src/lib/corelib/tools/tools.pri b/src/lib/corelib/tools/tools.pri
index bb3a55f12..f9c6be9a5 100644
--- a/src/lib/corelib/tools/tools.pri
+++ b/src/lib/corelib/tools/tools.pri
@@ -21,6 +21,7 @@ HEADERS += \
     $$PWD/generateoptions.h \
     $$PWD/id.h \
     $$PWD/iosutils.h \
+    $$PWD/joblimits.h \
     $$PWD/jsliterals.h \
     $$PWD/launcherinterface.h \
     $$PWD/launcherpackets.h \
@@ -75,6 +76,7 @@ SOURCES += \
     $$PWD/filetime.cpp \
     $$PWD/generateoptions.cpp \
     $$PWD/id.cpp \
+    $$PWD/joblimits.cpp \
     $$PWD/jsliterals.cpp \
     $$PWD/launcherinterface.cpp \
     $$PWD/launcherpackets.cpp \
@@ -124,6 +126,7 @@ osx {
         $$PWD/error.h \
         $$PWD/generateoptions.h \
         $$PWD/installoptions.h \
+        $$PWD/joblimits.h \
         $$PWD/preferences.h \
         $$PWD/processresult.h \
         $$PWD/profile.h \