diff options
author | Christian Kandeler <christian.kandeler@qt.io> | 2018-07-24 12:44:28 +0200 |
---|---|---|
committer | Christian Kandeler <christian.kandeler@qt.io> | 2018-08-08 11:18:45 +0000 |
commit | 0bc341e3dcba1f561a685d72cfa0f4b1a7f697dd (patch) | |
tree | bc79d5bc3f8bbbda56b72b0825c70bb71127d863 /src/lib | |
parent | 43e3b7ad7e5e68bae839feb74cdbcde48424c065 (diff) |
Add support for job pools
Commands can now be assigned to an arbitrary job pool and a limit for
the number of concurrently running jobs in such pools can be provided in
a number of ways:
- via the build command line: qbs --job-limits linker:1
- via the settings: qbs config preferences.jobLimit.linker 1
- in a project file: JobLimit {
jobPool: "linker";
jobCount: 1
}
We provide two job pools ourselves with the cpp module: "compiler" and
"linker".
[ChangeLog] Added the concept of job pools for limiting concurrent
execution of commands by type
Task-number: QBS-743
Change-Id: Ib3f361dbc73093e342bf0eba0daf2079a2b3a8ce
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@qt.io>
Reviewed-by: Joerg Bornemann <joerg.bornemann@qt.io>
Diffstat (limited to 'src/lib')
24 files changed, 569 insertions, 9 deletions
diff --git a/src/lib/corelib/buildgraph/executor.cpp b/src/lib/corelib/buildgraph/executor.cpp index f03529eec..368fa6ccb 100644 --- a/src/lib/corelib/buildgraph/executor.cpp +++ b/src/lib/corelib/buildgraph/executor.cpp @@ -61,10 +61,12 @@ #include <logging/translator.h> #include <tools/error.h> #include <tools/fileinfo.h> +#include <tools/preferences.h> #include <tools/profiling.h> #include <tools/progressobserver.h> #include <tools/qbsassert.h> #include <tools/qttools.h> +#include <tools/settings.h> #include <tools/stringconstants.h> #include <QtCore/qdir.h> @@ -230,6 +232,9 @@ void Executor::doBuild() m_tagsNeededForFilesToConsider.clear(); m_productsOfFilesToConsider.clear(); m_artifactsRemovedFromDisk.clear(); + m_jobCountPerPool.clear(); + + setupJobLimits(); // TODO: The "filesToConsider" thing is badly designed; we should know exactly which artifact // it is. Remove this from the BuildOptions class and introduce Project::buildSomeFiles() @@ -346,6 +351,7 @@ void Executor::updateLeaves(BuildGraphNode *node, NodeSet &seenNodes) bool Executor::scheduleJobs() { QBS_CHECK(m_state == ExecutorRunning); + std::vector<BuildGraphNode *> delayedLeaves; while (!m_leaves.empty() && !m_availableJobs.empty()) { BuildGraphNode * const nodeToBuild = m_leaves.top(); m_leaves.pop(); @@ -355,9 +361,17 @@ bool Executor::scheduleJobs() QBS_ASSERT(!"untouched node in leaves list", qDebug("%s", qPrintable(nodeToBuild->toString()))); break; - case BuildGraphNode::Buildable: - // This is the only state in which we want to build a node. - nodeToBuild->accept(this); + case BuildGraphNode::Buildable: // This is the only state in which we want to build a node. + // TODO: It's a bit annoying that we have to check this here already, when we + // don't know whether the transformer needs to run at all. Investigate + // moving the whole job allocation logic to runTransformer(). + if (schedulingBlockedByJobLimit(nodeToBuild)) { + qCDebug(lcExec).noquote() << "node delayed due to occupied job pool:" + << nodeToBuild->toString(); + delayedLeaves.push_back(nodeToBuild); + } else { + nodeToBuild->accept(this); + } break; case BuildGraphNode::Building: qCDebug(lcExec).noquote() << nodeToBuild->toString(); @@ -369,9 +383,50 @@ bool Executor::scheduleJobs() break; } } + for (BuildGraphNode * const delayedLeaf : delayedLeaves) + m_leaves.push(delayedLeaf); return !m_leaves.empty() || !m_processingJobs.empty(); } +bool Executor::schedulingBlockedByJobLimit(const BuildGraphNode *node) +{ + if (node->type() != BuildGraphNode::ArtifactNodeType) + return false; + const Artifact * const artifact = static_cast<const Artifact *>(node); + if (artifact->artifactType == Artifact::SourceFile) + return false; + + const Transformer * const transformer = artifact->transformer.get(); + for (const QString &jobPool : transformer->jobPools()) { + const int currentJobCount = m_jobCountPerPool[jobPool]; + if (currentJobCount == 0) + continue; + const auto jobLimitIsExceeded = [currentJobCount, jobPool, this](const Transformer *t) { + const int maxJobCount = m_jobLimitsPerProduct.at(t->product().get()) + .getLimit(jobPool); + return maxJobCount > 0 && currentJobCount >= maxJobCount; + }; + + // Different products can set different limits. The effective limit is the minimum of what + // is set in this transformer's product and in the products of all currently + // running transformers. + if (jobLimitIsExceeded(transformer)) + return true; + for (const ExecutorJob * const runningJob : m_processingJobs.keys()) { + if (!runningJob->jobPools().contains(jobPool)) + continue; + const Transformer * const runningTransformer = runningJob->transformer(); + if (!runningTransformer) + continue; // This can happen if the ExecutorJob has already finished. + if (runningTransformer->product() == transformer->product()) + continue; // We have already checked this product's job limit. + if (jobLimitIsExceeded(runningTransformer)) + return true; + } + } + return false; +} + bool Executor::isUpToDate(Artifact *artifact) const { QBS_CHECK(artifact->artifactType == Artifact::Generated); @@ -503,6 +558,7 @@ void Executor::finishJob(ExecutorJob *job, bool success) const TransformerPtr transformer = it.value(); m_processingJobs.erase(it); m_availableJobs.push_back(job); + updateJobCounts(transformer.get(), -1); if (success) { m_project->buildData->setDirty(); for (Artifact * const artifact : qAsConst(transformer->outputs)) { @@ -630,6 +686,30 @@ bool Executor::transformerHasMatchingInputFiles(const TransformerConstPtr &trans return false; } +void Executor::setupJobLimits() +{ + Settings settings(m_buildOptions.settingsDirectory()); + for (const ResolvedProductConstPtr &p : m_productsToBuild) { + const Preferences prefs(&settings, p->profile()); + const JobLimits &jobLimitsFromSettings = prefs.jobLimits(); + JobLimits effectiveJobLimits; + if (m_buildOptions.projectJobLimitsTakePrecedence()) { + effectiveJobLimits.update(jobLimitsFromSettings).update(m_buildOptions.jobLimits()) + .update(p->jobLimits); + } else { + effectiveJobLimits.update(p->jobLimits).update(jobLimitsFromSettings) + .update(m_buildOptions.jobLimits()); + } + m_jobLimitsPerProduct.insert(std::make_pair(p.get(), effectiveJobLimits)); + } +} + +void Executor::updateJobCounts(const Transformer *transformer, int diff) +{ + for (const QString &jobPool : transformer->jobPools()) + m_jobCountPerPool[jobPool] += diff; +} + void Executor::cancelJobs() { if (m_state == ExecutorCanceling) @@ -923,6 +1003,7 @@ void Executor::runTransformer(const TransformerPtr &transformer) for (Artifact * const artifact : qAsConst(transformer->outputs)) artifact->buildState = BuildGraphNode::Building; m_processingJobs.insert(job, transformer); + updateJobCounts(transformer.get(), 1); job->run(transformer.get()); } diff --git a/src/lib/corelib/buildgraph/executor.h b/src/lib/corelib/buildgraph/executor.h index 7225c0ace..4b4951e1e 100644 --- a/src/lib/corelib/buildgraph/executor.h +++ b/src/lib/corelib/buildgraph/executor.h @@ -154,6 +154,10 @@ private: bool artifactHasMatchingOutputTags(const Artifact *artifact) const; bool transformerHasMatchingInputFiles(const TransformerConstPtr &transformer) const; + void setupJobLimits(); + void updateJobCounts(const Transformer *transformer, int diff); + bool schedulingBlockedByJobLimit(const BuildGraphNode *node); + typedef QHash<ExecutorJob *, TransformerPtr> JobMap; JobMap m_processingJobs; @@ -169,6 +173,8 @@ private: std::vector<ResolvedProductPtr> m_allProducts; std::unordered_map<QString, const ResolvedProduct *> m_productsByName; std::unordered_map<QString, const ResolvedProject *> m_projectsByName; + std::unordered_map<QString, int> m_jobCountPerPool; + std::unordered_map<const ResolvedProduct *, JobLimits> m_jobLimitsPerProduct; NodeSet m_roots; Leaves m_leaves; InputArtifactScannerContext *m_inputArtifactScanContext; diff --git a/src/lib/corelib/buildgraph/executorjob.cpp b/src/lib/corelib/buildgraph/executorjob.cpp index e90b498f6..79f17377d 100644 --- a/src/lib/corelib/buildgraph/executorjob.cpp +++ b/src/lib/corelib/buildgraph/executorjob.cpp @@ -113,6 +113,7 @@ void ExecutorJob::run(Transformer *t) m_processCommandExecutor->setProcessEnvironment( (*t->outputs.cbegin())->product->buildEnvironment); m_transformer = t; + m_jobPools = t->jobPools(); runNextCommand(); } @@ -171,6 +172,7 @@ void ExecutorJob::setFinished() void ExecutorJob::reset() { m_transformer = nullptr; + m_jobPools.clear(); m_currentCommandExecutor = nullptr; m_currentCommandIdx = -1; m_error.clear(); diff --git a/src/lib/corelib/buildgraph/executorjob.h b/src/lib/corelib/buildgraph/executorjob.h index b1701a11d..e28d42f7f 100644 --- a/src/lib/corelib/buildgraph/executorjob.h +++ b/src/lib/corelib/buildgraph/executorjob.h @@ -43,8 +43,10 @@ #include <language/forward_decls.h> #include <tools/commandechomode.h> #include <tools/error.h> +#include <tools/set.h> #include <QtCore/qobject.h> +#include <QtCore/qstring.h> namespace qbs { class CodeLocation; @@ -71,6 +73,8 @@ public: void setEchoMode(CommandEchoMode echoMode); void run(Transformer *t); void cancel(); + const Transformer *transformer() const { return m_transformer; } + Set<QString> jobPools() const { return m_jobPools; } signals: void reportCommandDescription(const QString &highlight, const QString &message); @@ -88,6 +92,7 @@ private: ProcessCommandExecutor *m_processCommandExecutor; JsCommandExecutor *m_jsCommandExecutor; Transformer *m_transformer; + Set<QString> m_jobPools; int m_currentCommandIdx; ErrorInfo m_error; }; diff --git a/src/lib/corelib/buildgraph/rulecommands.cpp b/src/lib/corelib/buildgraph/rulecommands.cpp index c2aad1ce1..f7620dbf8 100644 --- a/src/lib/corelib/buildgraph/rulecommands.cpp +++ b/src/lib/corelib/buildgraph/rulecommands.cpp @@ -103,6 +103,7 @@ bool AbstractCommand::equals(const AbstractCommand *other) const && m_highlight == other->m_highlight && m_ignoreDryRun == other->m_ignoreDryRun && m_silent == other->m_silent + && m_jobPool == other->m_jobPool && m_properties == other->m_properties; } @@ -113,6 +114,7 @@ void AbstractCommand::fillFromScriptValue(const QScriptValue *scriptValue, const m_highlight = scriptValue->property(highlightProperty()).toString(); m_ignoreDryRun = scriptValue->property(ignoreDryRunProperty()).toBool(); m_silent = scriptValue->property(silentProperty()).toBool(); + m_jobPool = scriptValue->property(StringConstants::jobPoolProperty()).toString(); m_codeLocation = codeLocation; m_predefinedProperties @@ -120,6 +122,7 @@ void AbstractCommand::fillFromScriptValue(const QScriptValue *scriptValue, const << extendedDescriptionProperty() << highlightProperty() << ignoreDryRunProperty() + << StringConstants::jobPoolProperty() << silentProperty(); } diff --git a/src/lib/corelib/buildgraph/rulecommands.h b/src/lib/corelib/buildgraph/rulecommands.h index 7583de2b6..02d1a02a5 100644 --- a/src/lib/corelib/buildgraph/rulecommands.h +++ b/src/lib/corelib/buildgraph/rulecommands.h @@ -81,6 +81,7 @@ public: const QString highlight() const { return m_highlight; } bool ignoreDryRun() const { return m_ignoreDryRun; } bool isSilent() const { return m_silent; } + QString jobPool() const { return m_jobPool; } CodeLocation codeLocation() const { return m_codeLocation; } const QVariantMap &properties() const { return m_properties; } @@ -98,7 +99,8 @@ private: template<PersistentPool::OpType opType> void serializationOp(PersistentPool &pool) { pool.serializationOp<opType>(m_description, m_extendedDescription, m_highlight, - m_ignoreDryRun, m_silent, m_codeLocation, m_properties); + m_ignoreDryRun, m_silent, m_codeLocation, m_jobPool, + m_properties); } QString m_description; @@ -107,6 +109,7 @@ private: bool m_ignoreDryRun; bool m_silent; CodeLocation m_codeLocation; + QString m_jobPool; QVariantMap m_properties; }; diff --git a/src/lib/corelib/buildgraph/transformer.cpp b/src/lib/corelib/buildgraph/transformer.cpp index df371e1fe..4569b9955 100644 --- a/src/lib/corelib/buildgraph/transformer.cpp +++ b/src/lib/corelib/buildgraph/transformer.cpp @@ -305,5 +305,15 @@ void Transformer::rescueChangeTrackingData(const TransformerConstPtr &other) exportedModulesAccessedInCommands = other->exportedModulesAccessedInCommands; } +Set<QString> Transformer::jobPools() const +{ + Set<QString> pools; + for (const AbstractCommandPtr &c : commands.commands()) { + if (!c->jobPool().isEmpty()) + pools.insert(c->jobPool()); + } + return pools; +} + } // namespace Internal } // namespace qbs diff --git a/src/lib/corelib/buildgraph/transformer.h b/src/lib/corelib/buildgraph/transformer.h index ce66a976e..2f6a8e56d 100644 --- a/src/lib/corelib/buildgraph/transformer.h +++ b/src/lib/corelib/buildgraph/transformer.h @@ -102,6 +102,8 @@ public: const QScriptValueList &args); void rescueChangeTrackingData(const TransformerConstPtr &other); + Set<QString> jobPools() const; + template<PersistentPool::OpType opType> void completeSerializationOp(PersistentPool &pool) { pool.serializationOp<opType>(rule, inputs, outputs, explicitlyDependsOn, diff --git a/src/lib/corelib/corelib.qbs b/src/lib/corelib/corelib.qbs index 93509763b..c947cb484 100644 --- a/src/lib/corelib/corelib.qbs +++ b/src/lib/corelib/corelib.qbs @@ -397,6 +397,7 @@ QbsLibrary { "id.cpp", "id.h", "iosutils.h", + "joblimits.cpp", "jsliterals.cpp", "jsliterals.h", "installoptions.cpp", @@ -467,6 +468,7 @@ QbsLibrary { "error.h", "generateoptions.h", "installoptions.h", + "joblimits.h", "preferences.h", "processresult.h", "profile.h", diff --git a/src/lib/corelib/language/builtindeclarations.cpp b/src/lib/corelib/language/builtindeclarations.cpp index 4be405112..4886675d0 100644 --- a/src/lib/corelib/language/builtindeclarations.cpp +++ b/src/lib/corelib/language/builtindeclarations.cpp @@ -67,6 +67,7 @@ BuiltinDeclarations::BuiltinDeclarations() { QLatin1String("Export"), ItemType::Export }, { QLatin1String("FileTagger"), ItemType::FileTagger }, { QLatin1String("Group"), ItemType::Group }, + { QLatin1String("JobLimit"), ItemType::JobLimit }, { QLatin1String("Module"), ItemType::Module }, { QLatin1String("Parameter"), ItemType::Parameter }, { QLatin1String("Parameters"), ItemType::Parameters }, @@ -87,6 +88,7 @@ BuiltinDeclarations::BuiltinDeclarations() addExportItem(); addFileTaggerItem(); addGroupItem(); + addJobLimitItem(); addModuleItem(); addProbeItem(); addProductItem(); @@ -298,6 +300,15 @@ void BuiltinDeclarations::addGroupItem() insert(item); } +void BuiltinDeclarations::addJobLimitItem() +{ + ItemDeclaration item(ItemType::JobLimit); + item << conditionProperty(); + item << PropertyDeclaration(StringConstants::jobPoolProperty(), PropertyDeclaration::String); + item << PropertyDeclaration(StringConstants::jobCountProperty(), PropertyDeclaration::Integer); + insert(item); +} + void BuiltinDeclarations::addModuleItem() { ItemDeclaration item = moduleLikeItem(ItemType::Module); @@ -312,6 +323,7 @@ ItemDeclaration BuiltinDeclarations::moduleLikeItem(ItemType type) << ItemType::Group << ItemType::Depends << ItemType::FileTagger + << ItemType::JobLimit << ItemType::Rule << ItemType::Parameter << ItemType::Probe @@ -360,6 +372,7 @@ void BuiltinDeclarations::addProductItem() << ItemType::Depends << ItemType::Group << ItemType::FileTagger + << ItemType::JobLimit << ItemType::Export << ItemType::Probe << ItemType::Profile @@ -429,6 +442,7 @@ void BuiltinDeclarations::addProjectItem() << ItemType::Profile << ItemType::Probe << ItemType::FileTagger + << ItemType::JobLimit << ItemType::Rule); item << nameProperty(); item << conditionProperty(); diff --git a/src/lib/corelib/language/builtindeclarations.h b/src/lib/corelib/language/builtindeclarations.h index c75475df7..ff16b395a 100644 --- a/src/lib/corelib/language/builtindeclarations.h +++ b/src/lib/corelib/language/builtindeclarations.h @@ -76,6 +76,7 @@ private: void addExportItem(); void addFileTaggerItem(); void addGroupItem(); + void addJobLimitItem(); void addModuleItem(); static ItemDeclaration moduleLikeItem(ItemType type); void addProbeItem(); diff --git a/src/lib/corelib/language/itemtype.h b/src/lib/corelib/language/itemtype.h index c0e76c94b..324a1fb87 100644 --- a/src/lib/corelib/language/itemtype.h +++ b/src/lib/corelib/language/itemtype.h @@ -53,6 +53,7 @@ enum class ItemType { Export, FileTagger, Group, + JobLimit, Module, Parameter, Parameters, diff --git a/src/lib/corelib/language/language.h b/src/lib/corelib/language/language.h index 580b3810c..06a9af54a 100644 --- a/src/lib/corelib/language/language.h +++ b/src/lib/corelib/language/language.h @@ -49,6 +49,7 @@ #include <buildgraph/forward_decls.h> #include <tools/codelocation.h> #include <tools/filetime.h> +#include <tools/joblimits.h> #include <tools/persistence.h> #include <tools/set.h> #include <tools/weakpointer.h> @@ -571,6 +572,7 @@ public: std::vector<ResolvedProductPtr> dependencies; QHash<ResolvedProductConstPtr, QVariantMap> dependencyParameters; std::vector<FileTaggerConstPtr> fileTaggers; + JobLimits jobLimits; std::vector<ResolvedModulePtr> modules; QHash<ResolvedModuleConstPtr, QVariantMap> moduleParameters; std::vector<ResolvedScannerConstPtr> scanners; @@ -628,7 +630,8 @@ private: missingSourceFiles, location, productProperties, moduleProperties, rules, dependencies, dependencyParameters, fileTaggers, modules, moduleParameters, scanners, groups, - artifactProperties, probes, exportedModule, buildData); + artifactProperties, probes, exportedModule, buildData, + jobLimits); } QHash<QString, QString> m_executablePathCache; diff --git a/src/lib/corelib/language/projectresolver.cpp b/src/lib/corelib/language/projectresolver.cpp index ec29e00c0..27829e378 100644 --- a/src/lib/corelib/language/projectresolver.cpp +++ b/src/lib/corelib/language/projectresolver.cpp @@ -56,6 +56,7 @@ #include <logging/translator.h> #include <tools/error.h> #include <tools/fileinfo.h> +#include <tools/joblimits.h> #include <tools/jsliterals.h> #include <tools/profiling.h> #include <tools/progressobserver.h> @@ -89,6 +90,7 @@ struct ProjectResolver::ProjectContext ResolvedProjectPtr project; std::vector<FileTaggerConstPtr> fileTaggers; std::vector<RulePtr> rules; + JobLimits jobLimits; ResolvedModulePtr dummyModule; }; @@ -106,6 +108,7 @@ struct ProjectResolver::ProductContext struct ProjectResolver::ModuleContext { ResolvedModulePtr module; + JobLimits jobLimits; }; class CancelException { }; @@ -333,6 +336,7 @@ void ProjectResolver::resolveProjectFully(Item *item, ProjectResolver::ProjectCo { ItemType::Product, &ProjectResolver::resolveProduct }, { ItemType::Probe, &ProjectResolver::ignoreItem }, { ItemType::FileTagger, &ProjectResolver::resolveFileTagger }, + { ItemType::JobLimit, &ProjectResolver::resolveJobLimit }, { ItemType::Rule, &ProjectResolver::resolveRule }, { ItemType::PropertyOptions, &ProjectResolver::ignoreItem } }; @@ -494,6 +498,7 @@ void ProjectResolver::resolveProductFully(Item *item, ProjectContext *projectCon { ItemType::Depends, &ProjectResolver::ignoreItem }, { ItemType::Rule, &ProjectResolver::resolveRule }, { ItemType::FileTagger, &ProjectResolver::resolveFileTagger }, + { ItemType::JobLimit, &ProjectResolver::resolveJobLimit }, { ItemType::Group, &ProjectResolver::resolveGroup }, { ItemType::Product, &ProjectResolver::resolveShadowProduct }, { ItemType::Export, &ProjectResolver::resolveExport }, @@ -504,6 +509,11 @@ void ProjectResolver::resolveProductFully(Item *item, ProjectContext *projectCon for (Item * const child : qAsConst(subItems)) callItemFunction(mapping, child, projectContext); + for (const ProjectContext *p = projectContext; p; p = p->parentContext) { + JobLimits tempLimits = p->jobLimits; + product->jobLimits = tempLimits.update(product->jobLimits); + } + resolveModules(item, projectContext); for (const FileTag &t : qAsConst(product->fileTags)) @@ -512,12 +522,19 @@ void ProjectResolver::resolveProductFully(Item *item, ProjectContext *projectCon void ProjectResolver::resolveModules(const Item *item, ProjectContext *projectContext) { + JobLimits jobLimits; for (const Item::Module &m : item->modules()) - resolveModule(m.name, m.item, m.isProduct, m.parameters, projectContext); + resolveModule(m.name, m.item, m.isProduct, m.parameters, jobLimits, projectContext); + for (int i = 0; i < jobLimits.count(); ++i) { + const JobLimit &moduleJobLimit = jobLimits.jobLimitAt(i); + if (m_productContext->product->jobLimits.getLimit(moduleJobLimit.pool()) == -1) + m_productContext->product->jobLimits.setJobLimit(moduleJobLimit); + } } void ProjectResolver::resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct, - const QVariantMap ¶meters, ProjectContext *projectContext) + const QVariantMap ¶meters, JobLimits &jobLimits, + ProjectContext *projectContext) { checkCancelation(); if (!item->isPresentModule()) @@ -550,6 +567,7 @@ void ProjectResolver::resolveModule(const QualifiedId &moduleName, Item *item, b { ItemType::Group, &ProjectResolver::ignoreItem }, { ItemType::Rule, &ProjectResolver::resolveRule }, { ItemType::FileTagger, &ProjectResolver::resolveFileTagger }, + { ItemType::JobLimit, &ProjectResolver::resolveJobLimit }, { ItemType::Scanner, &ProjectResolver::resolveScanner }, { ItemType::PropertyOptions, &ProjectResolver::ignoreItem }, { ItemType::Depends, &ProjectResolver::ignoreItem }, @@ -559,6 +577,12 @@ void ProjectResolver::resolveModule(const QualifiedId &moduleName, Item *item, b }; for (Item *child : item->children()) callItemFunction(mapping, child, projectContext); + for (int i = 0; i < moduleContext.jobLimits.count(); ++i) { + const JobLimit &newJobLimit = moduleContext.jobLimits.jobLimitAt(i); + const int oldLimit = jobLimits.getLimit(newJobLimit.pool()); + if (oldLimit == -1 || oldLimit > newJobLimit.limit()) + jobLimits.setJobLimit(newJobLimit); + } m_moduleContext = oldModuleContext; } @@ -1314,6 +1338,35 @@ void ProjectResolver::resolveFileTagger(Item *item, ProjectContext *projectConte fileTaggers.push_back(FileTagger::create(patterns, fileTags, priority)); } +void ProjectResolver::resolveJobLimit(Item *item, ProjectResolver::ProjectContext *projectContext) +{ + if (!m_evaluator->boolValue(item, StringConstants::conditionProperty())) + return; + const QString jobPool = m_evaluator->stringValue(item, StringConstants::jobPoolProperty()); + if (jobPool.isEmpty()) + throw ErrorInfo(Tr::tr("A JobLimit item needs to have a non-empty '%1' property.") + .arg(StringConstants::jobPoolProperty()), item->location()); + bool jobCountWasSet; + const int jobCount = m_evaluator->intValue(item, StringConstants::jobCountProperty(), -1, + &jobCountWasSet); + if (!jobCountWasSet) { + throw ErrorInfo(Tr::tr("A JobLimit item needs to have a '%1' property.") + .arg(StringConstants::jobCountProperty()), item->location()); + } + if (jobCount < 0) { + throw ErrorInfo(Tr::tr("A JobLimit item must have a non-negative '%1' property.") + .arg(StringConstants::jobCountProperty()), item->location()); + } + JobLimits &jobLimits = m_moduleContext + ? m_moduleContext->jobLimits + : m_productContext ? m_productContext->product->jobLimits + : projectContext->jobLimits; + JobLimit jobLimit(jobPool, jobCount); + const int oldLimit = jobLimits.getLimit(jobPool); + if (oldLimit == -1 || oldLimit > jobCount) + jobLimits.setJobLimit(jobLimit); +} + void ProjectResolver::resolveScanner(Item *item, ProjectResolver::ProjectContext *projectContext) { checkCancelation(); diff --git a/src/lib/corelib/language/projectresolver.h b/src/lib/corelib/language/projectresolver.h index 660bf6259..63a083fef 100644 --- a/src/lib/corelib/language/projectresolver.h +++ b/src/lib/corelib/language/projectresolver.h @@ -56,6 +56,7 @@ #include <vector> namespace qbs { +class JobLimits; namespace Internal { class Evaluator; @@ -101,7 +102,8 @@ private: void resolveProductFully(Item *item, ProjectContext *projectContext); void resolveModules(const Item *item, ProjectContext *projectContext); void resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct, - const QVariantMap ¶meters, ProjectContext *projectContext); + const QVariantMap ¶meters, JobLimits &jobLimits, + ProjectContext *projectContext); void gatherProductTypes(ResolvedProduct *product, Item *item); QVariantMap resolveAdditionalModuleProperties(const Item *group, const QVariantMap ¤tValues); @@ -117,6 +119,7 @@ private: const QStringList &namePrefix, QualifiedIdSet *seenBindings); void resolveFileTagger(Item *item, ProjectContext *projectContext); + void resolveJobLimit(Item *item, ProjectContext *projectContext); void resolveScanner(Item *item, ProjectContext *projectContext); void resolveProductDependencies(const ProjectContext &projectContext); void postProcess(const ResolvedProductPtr &product, ProjectContext *projectContext) const; diff --git a/src/lib/corelib/tools/buildoptions.cpp b/src/lib/corelib/tools/buildoptions.cpp index ac8e39867..5507e0842 100644 --- a/src/lib/corelib/tools/buildoptions.cpp +++ b/src/lib/corelib/tools/buildoptions.cpp @@ -58,6 +58,8 @@ public: QStringList changedFiles; QStringList filesToConsider; QStringList activeFileTags; + JobLimits jobLimits; + QString settingsDir; int maxJobCount; bool dryRun; bool keepGoing; @@ -68,6 +70,7 @@ public: bool install; bool removeExistingInstallation; bool onlyExecuteRules; + bool jobLimitsFromProjectTakePrecedence = false; }; } // namespace Internal @@ -189,6 +192,44 @@ void BuildOptions::setMaxJobCount(int jobCount) } /*! + * \brief The base directory for qbs settings. + * This value is used to locate profiles and preferences. + */ +QString BuildOptions::settingsDirectory() const +{ + return d->settingsDir; +} + +/*! + * \brief Sets the base directory for qbs settings. + * \param settingsBaseDir Will be used to locate profiles and preferences. + */ +void BuildOptions::setSettingsDirectory(const QString &settingsBaseDir) +{ + d->settingsDir = settingsBaseDir; +} + +JobLimits BuildOptions::jobLimits() const +{ + return d->jobLimits; +} + +void BuildOptions::setJobLimits(const JobLimits &jobLimits) +{ + d->jobLimits = jobLimits; +} + +bool BuildOptions::projectJobLimitsTakePrecedence() const +{ + return d->jobLimitsFromProjectTakePrecedence; +} + +void BuildOptions::setProjectJobLimitsTakePrecedence(bool toggle) +{ + d->jobLimitsFromProjectTakePrecedence = toggle; +} + +/*! * \brief Returns true iff qbs will not actually execute any commands, but just show what * would happen. * The default is false. diff --git a/src/lib/corelib/tools/buildoptions.h b/src/lib/corelib/tools/buildoptions.h index 630a6aa22..cea89d0ea 100644 --- a/src/lib/corelib/tools/buildoptions.h +++ b/src/lib/corelib/tools/buildoptions.h @@ -42,6 +42,7 @@ #include "qbs_export.h" #include "commandechomode.h" +#include "joblimits.h" #include <QtCore/qshareddata.h> @@ -73,6 +74,15 @@ public: int maxJobCount() const; void setMaxJobCount(int jobCount); + QString settingsDirectory() const; + void setSettingsDirectory(const QString &settingsBaseDir); + + JobLimits jobLimits() const; + void setJobLimits(const JobLimits &jobLimits); + + bool projectJobLimitsTakePrecedence() const; + void setProjectJobLimitsTakePrecedence(bool toggle); + bool dryRun() const; void setDryRun(bool dryRun); diff --git a/src/lib/corelib/tools/joblimits.cpp b/src/lib/corelib/tools/joblimits.cpp new file mode 100644 index 000000000..3b1fde83d --- /dev/null +++ b/src/lib/corelib/tools/joblimits.cpp @@ -0,0 +1,185 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "joblimits.h" + +#include <tools/persistence.h> + +#include <utility> +#include <vector> + +namespace qbs { +namespace Internal { + +static int transformLimit(int limitFromUser) +{ + return limitFromUser == 0 + ? std::numeric_limits<int>::max() + : limitFromUser < -1 ? -1 + : limitFromUser; +} + +class JobLimitPrivate : public QSharedData +{ +public: + JobLimitPrivate(const QString &pool, int limit) + : jobLimit(std::make_pair(pool, transformLimit(limit))) + { + } + template<PersistentPool::OpType opType> void serializationOp(PersistentPool &pool) + { + pool.serializationOp<opType>(jobLimit); + } + std::pair<QString, int> jobLimit; +}; + +class JobLimitsPrivate : public QSharedData +{ +public: + template<PersistentPool::OpType opType> void serializationOp(PersistentPool &pool) + { + pool.serializationOp<opType>(jobLimits); + } + std::vector<JobLimit> jobLimits; +}; + +} // namespace Internal + +JobLimit::JobLimit() : JobLimit(QString(), -1) +{ +} +JobLimit::JobLimit(const QString &pool, int limit) : d(new Internal::JobLimitPrivate(pool, limit)) +{ +} +JobLimit::JobLimit(const JobLimit &other) : d(other.d) { } +JobLimit &JobLimit::operator=(const JobLimit &other) +{ + d = other.d; + return *this; +} +JobLimit::~JobLimit() {} +QString JobLimit::pool() const { return d->jobLimit.first; } +int JobLimit::limit() const { return d->jobLimit.second; } + +void JobLimit::load(Internal::PersistentPool &pool) +{ + d->serializationOp<Internal::PersistentPool::Load>(pool); +} + +void JobLimit::store(Internal::PersistentPool &pool) +{ + d->serializationOp<Internal::PersistentPool::Store>(pool); +} + +JobLimits::JobLimits() : d(new Internal::JobLimitsPrivate) { } +JobLimits::JobLimits(const JobLimits &other) : d(other.d) { } +JobLimits &JobLimits::operator=(const JobLimits &other) +{ + d = other.d; + return *this; +} +JobLimits::~JobLimits() {} + +void JobLimits::setJobLimit(const JobLimit &limit) +{ + for (std::size_t i = 0; i < d->jobLimits.size(); ++i) { + JobLimit ¤tLimit = d->jobLimits.at(i); + if (currentLimit.pool() == limit.pool()) { + if (currentLimit.limit() != limit.limit()) + currentLimit = limit; + return; + } + } + d->jobLimits.push_back(limit); +} + +void JobLimits::setJobLimit(const QString &pool, int limit) +{ + setJobLimit(JobLimit(pool, limit)); +} + +int JobLimits::getLimit(const QString &pool) const +{ + for (const JobLimit &l : d->jobLimits) { + if (l.pool() == pool) + return l.limit(); + } + return -1; +} + +bool JobLimits::isEmpty() const +{ + return d->jobLimits.empty(); +} + +int JobLimits::count() const +{ + return d->jobLimits.size(); +} + +JobLimit JobLimits::jobLimitAt(int i) const +{ + return d->jobLimits.at(i); +} + +JobLimits &JobLimits::update(const JobLimits &other) +{ + if (isEmpty()) { + *this = other; + } else { + for (int i = 0; i < other.count(); ++i) { + const JobLimit &l = other.jobLimitAt(i); + if (l.limit() != -1) + setJobLimit(l); + } + } + return *this; +} + +void JobLimits::load(Internal::PersistentPool &pool) +{ + d->serializationOp<Internal::PersistentPool::Load>(pool); +} + +void JobLimits::store(Internal::PersistentPool &pool) +{ + d->serializationOp<Internal::PersistentPool::Store>(pool); +} + +} // namespace qbs diff --git a/src/lib/corelib/tools/joblimits.h b/src/lib/corelib/tools/joblimits.h new file mode 100644 index 000000000..de95f5513 --- /dev/null +++ b/src/lib/corelib/tools/joblimits.h @@ -0,0 +1,102 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#ifndef QBS_JOB_LIMITS_H +#define QBS_JOB_LIMITS_H + +#include "qbs_export.h" + +#include <QtCore/qshareddata.h> + +QT_BEGIN_NAMESPACE +class QString; +QT_END_NAMESPACE + +namespace qbs { +namespace Internal { +class JobLimitPrivate; +class JobLimitsPrivate; +class PersistentPool; +} + +class QBS_EXPORT JobLimit +{ +public: + JobLimit(); + JobLimit(const QString &pool, int limit); + JobLimit(const JobLimit &other); + JobLimit &operator=(const JobLimit &other); + ~JobLimit(); + + QString pool() const; + int limit() const; + + void load(Internal::PersistentPool &pool); + void store(Internal::PersistentPool &pool); +private: + QSharedDataPointer<Internal::JobLimitPrivate> d; +}; + +class QBS_EXPORT JobLimits +{ +public: + JobLimits(); + JobLimits(const JobLimits &other); + JobLimits &operator=(const JobLimits &other); + ~JobLimits(); + + void setJobLimit(const JobLimit &limit); + void setJobLimit(const QString &pool, int limit); + int getLimit(const QString &pool) const; + bool hasLimit(const QString &pool) const { return getLimit(pool) != -1; } + bool isEmpty() const; + + int count() const; + JobLimit jobLimitAt(int i) const; + + JobLimits &update(const JobLimits &other); + + void load(Internal::PersistentPool &pool); + void store(Internal::PersistentPool &pool); +private: + QSharedDataPointer<Internal::JobLimitsPrivate> d; +}; + +} // namespace qbs + +#endif // include guard diff --git a/src/lib/corelib/tools/persistence.cpp b/src/lib/corelib/tools/persistence.cpp index 53c9dfac4..216dfad24 100644 --- a/src/lib/corelib/tools/persistence.cpp +++ b/src/lib/corelib/tools/persistence.cpp @@ -48,7 +48,7 @@ namespace qbs { namespace Internal { -static const char QBS_PERSISTENCE_MAGIC[] = "QBSPERSISTENCE-122"; +static const char QBS_PERSISTENCE_MAGIC[] = "QBSPERSISTENCE-123"; NoBuildGraphError::NoBuildGraphError(const QString &filePath) : ErrorInfo(Tr::tr("Build graph not found for configuration '%1'. Expected location was '%2'.") diff --git a/src/lib/corelib/tools/preferences.cpp b/src/lib/corelib/tools/preferences.cpp index 66803a0f5..12af4e9c7 100644 --- a/src/lib/corelib/tools/preferences.cpp +++ b/src/lib/corelib/tools/preferences.cpp @@ -124,6 +124,32 @@ QStringList Preferences::pluginPaths(const QString &baseDir) const return pathList(QLatin1String("pluginsPath"), baseDir + QLatin1String("/qbs/plugins")); } +/*! + * \brief Returns the per-pool job limits. + */ +JobLimits Preferences::jobLimits() const +{ + const QString prefix = QLatin1String("preferences.jobLimit"); + JobLimits limits; + for (const QString &key : m_settings->allKeysWithPrefix(prefix, Settings::allScopes())) { + limits.setJobLimit(key, m_settings->value(prefix + QLatin1Char('.') + key, + Settings::allScopes()).toInt()); + } + const QString fullPrefix = prefix + QLatin1Char('.'); + if (!m_profile.isEmpty()) { + Profile p(m_profile, m_settings, m_profileContents); + for (const QString &key : p.allKeys(Profile::KeySelectionRecursive)) { + if (!key.startsWith(fullPrefix)) + continue; + const QString jobPool = key.mid(fullPrefix.size()); + const int limit = p.value(key).toInt(); + if (limit >= 0) + limits.setJobLimit(jobPool, limit); + } + } + return limits; +} + QVariant Preferences::getPreference(const QString &key, const QVariant &defaultValue) const { static const QString keyPrefix = QLatin1String("preferences"); diff --git a/src/lib/corelib/tools/preferences.h b/src/lib/corelib/tools/preferences.h index 07f0edcd7..661b39d7f 100644 --- a/src/lib/corelib/tools/preferences.h +++ b/src/lib/corelib/tools/preferences.h @@ -42,6 +42,7 @@ #include "qbs_export.h" #include "commandechomode.h" +#include "joblimits.h" #include "settings.h" #include <QtCore/qstringlist.h> @@ -63,6 +64,7 @@ public: CommandEchoMode defaultEchoMode() const; QStringList searchPaths(const QString &baseDir = QString()) const; QStringList pluginPaths(const QString &baseDir = QString()) const; + JobLimits jobLimits() const; private: QVariant getPreference(const QString &key, const QVariant &defaultValue = QVariant()) const; diff --git a/src/lib/corelib/tools/stringconstants.h b/src/lib/corelib/tools/stringconstants.h index f2666e070..6fcf3002b 100644 --- a/src/lib/corelib/tools/stringconstants.h +++ b/src/lib/corelib/tools/stringconstants.h @@ -104,6 +104,8 @@ public: QBS_STRING_CONSTANT(installPrefixProperty, "installPrefix") QBS_STRING_CONSTANT(installDirProperty, "installDir") QBS_STRING_CONSTANT(installSourceBaseProperty, "installSourceBase") + QBS_STRING_CONSTANT(jobCountProperty, "jobCount") + QBS_STRING_CONSTANT(jobPoolProperty, "jobPool") QBS_STRING_CONSTANT(lengthProperty, "length") QBS_STRING_CONSTANT(limitToSubProjectProperty, "limitToSubProject") QBS_STRING_CONSTANT(minimumQbsVersionProperty, "minimumQbsVersion") diff --git a/src/lib/corelib/tools/tools.pri b/src/lib/corelib/tools/tools.pri index bb3a55f12..f9c6be9a5 100644 --- a/src/lib/corelib/tools/tools.pri +++ b/src/lib/corelib/tools/tools.pri @@ -21,6 +21,7 @@ HEADERS += \ $$PWD/generateoptions.h \ $$PWD/id.h \ $$PWD/iosutils.h \ + $$PWD/joblimits.h \ $$PWD/jsliterals.h \ $$PWD/launcherinterface.h \ $$PWD/launcherpackets.h \ @@ -75,6 +76,7 @@ SOURCES += \ $$PWD/filetime.cpp \ $$PWD/generateoptions.cpp \ $$PWD/id.cpp \ + $$PWD/joblimits.cpp \ $$PWD/jsliterals.cpp \ $$PWD/launcherinterface.cpp \ $$PWD/launcherpackets.cpp \ @@ -124,6 +126,7 @@ osx { $$PWD/error.h \ $$PWD/generateoptions.h \ $$PWD/installoptions.h \ + $$PWD/joblimits.h \ $$PWD/preferences.h \ $$PWD/processresult.h \ $$PWD/profile.h \ |