From 0bc341e3dcba1f561a685d72cfa0f4b1a7f697dd Mon Sep 17 00:00:00 2001 From: Christian Kandeler Date: Tue, 24 Jul 2018 12:44:28 +0200 Subject: Add support for job pools Commands can now be assigned to an arbitrary job pool and a limit for the number of concurrently running jobs in such pools can be provided in a number of ways: - via the build command line: qbs --job-limits linker:1 - via the settings: qbs config preferences.jobLimit.linker 1 - in a project file: JobLimit { jobPool: "linker"; jobCount: 1 } We provide two job pools ourselves with the cpp module: "compiler" and "linker". [ChangeLog] Added the concept of job pools for limiting concurrent execution of commands by type Task-number: QBS-743 Change-Id: Ib3f361dbc73093e342bf0eba0daf2079a2b3a8ce Reviewed-by: Oswald Buddenhagen Reviewed-by: Joerg Bornemann --- doc/howtos.qdoc | 49 ++++++ doc/reference/cli/builtin/cli-build.qdoc | 1 + doc/reference/cli/cli-options.qdocinc | 16 ++ doc/reference/commands.qdoc | 7 + doc/reference/items/language/group.qdoc | 2 +- doc/reference/items/language/joblimit.qdoc | 107 ++++++++++++ doc/reference/items/language/module.qdoc | 2 +- doc/reference/modules/cpp-module.qdoc | 25 +++ share/qbs/modules/cpp/GenericGCC.qbs | 1 + share/qbs/modules/cpp/gcc.js | 3 + share/qbs/modules/cpp/msvc.js | 2 + share/qbs/modules/cpp/windows-msvc.qbs | 3 + src/app/qbs/parser/commandlineoption.cpp | 53 ++++++ src/app/qbs/parser/commandlineoption.h | 26 +++ src/app/qbs/parser/commandlineoptionpool.cpp | 17 ++ src/app/qbs/parser/commandlineoptionpool.h | 2 + src/app/qbs/parser/commandlineparser.cpp | 4 + src/app/qbs/parser/parsercommand.cpp | 2 + src/lib/corelib/buildgraph/executor.cpp | 87 +++++++++- src/lib/corelib/buildgraph/executor.h | 6 + src/lib/corelib/buildgraph/executorjob.cpp | 2 + src/lib/corelib/buildgraph/executorjob.h | 5 + src/lib/corelib/buildgraph/rulecommands.cpp | 3 + src/lib/corelib/buildgraph/rulecommands.h | 5 +- src/lib/corelib/buildgraph/transformer.cpp | 10 ++ src/lib/corelib/buildgraph/transformer.h | 2 + src/lib/corelib/corelib.qbs | 2 + src/lib/corelib/language/builtindeclarations.cpp | 14 ++ src/lib/corelib/language/builtindeclarations.h | 1 + src/lib/corelib/language/itemtype.h | 1 + src/lib/corelib/language/language.h | 5 +- src/lib/corelib/language/projectresolver.cpp | 57 ++++++- src/lib/corelib/language/projectresolver.h | 5 +- src/lib/corelib/tools/buildoptions.cpp | 41 +++++ src/lib/corelib/tools/buildoptions.h | 10 ++ src/lib/corelib/tools/joblimits.cpp | 185 +++++++++++++++++++++ src/lib/corelib/tools/joblimits.h | 102 ++++++++++++ src/lib/corelib/tools/persistence.cpp | 2 +- src/lib/corelib/tools/preferences.cpp | 26 +++ src/lib/corelib/tools/preferences.h | 2 + src/lib/corelib/tools/stringconstants.h | 2 + src/lib/corelib/tools/tools.pri | 3 + tests/auto/auto.pro | 1 + tests/auto/auto.qbs | 1 + tests/auto/blackbox/blackbox-joblimits.pro | 18 ++ tests/auto/blackbox/blackbox-joblimits.qbs | 20 +++ .../testdata-joblimits/job-limits/job-limits.qbs | 93 +++++++++++ .../testdata-joblimits/job-limits/main.cpp | 65 ++++++++ tests/auto/blackbox/tst_blackboxjoblimits.cpp | 174 +++++++++++++++++++ 49 files changed, 1261 insertions(+), 11 deletions(-) create mode 100644 doc/reference/items/language/joblimit.qdoc create mode 100644 src/lib/corelib/tools/joblimits.cpp create mode 100644 src/lib/corelib/tools/joblimits.h create mode 100644 tests/auto/blackbox/blackbox-joblimits.pro create mode 100644 tests/auto/blackbox/blackbox-joblimits.qbs create mode 100644 tests/auto/blackbox/testdata-joblimits/job-limits/job-limits.qbs create mode 100644 tests/auto/blackbox/testdata-joblimits/job-limits/main.cpp create mode 100644 tests/auto/blackbox/tst_blackboxjoblimits.cpp diff --git a/doc/howtos.qdoc b/doc/howtos.qdoc index dd1b45130..7982ae6d4 100644 --- a/doc/howtos.qdoc +++ b/doc/howtos.qdoc @@ -44,6 +44,7 @@ \li \l{How do I create application bundles and frameworks on iOS, macOS, tvOS, and watchOS?} \li \l{How do I apply C/C++ preprocessor macros to only a subset of the files in my product?} \li \l{How do I make the state of my Git repository available to my source files?} + \li \l{How do I limit the number of concurrent jobs for the linker only?} \endlist \section1 How do I build a Qt-based project? @@ -366,4 +367,52 @@ This value is also available via the \l{vcs::repoState}{vcs.repoState} property. + + \section1 How do I limit the number of concurrent jobs for the linker only? + \target job-pool-howto + + While it is usually desirable to run as many compiler jobs as there are CPU cores, + the same is not true for linker jobs. The reason is that linkers are typically + I/O bound rather than CPU bound. When building large libraries, they also tend + to use up enormous amounts of memory. Therefore, we'd like to make sure that + only a few linkers are running at the same time without limiting other types + of jobs. In \QBS, this is achieved via \e{job pools}. There are several ways + to make use of them. + + Firstly, you can provide a limit via the command line: + \code + $ qbs --job-limits linker:4 + \endcode + The above call instructs \QBS to run at most four linker instances at the same + time, while leaving the general number of concurrent jobs at the default + value, which is derived from the number of CPU cores. + The \c linker string on the command line refers to the job pool of the same + name, which the \l{cpp-job-pools}{cpp module} assigns to all its commands that + invoke a linker. + + Secondly, you can set a limit via the settings, either generally + or for a specific profile: + \code + $ qbs config preferences.jobLimit.linker 4 + $ qbs config profiles.myprofile.preferences.jobLimit.linker 2 + \endcode + + And finally, you can also set the limit per project or per product, using a + \l JobLimit item: + \code + Product { + name: "my_huge_library" + JobLimit { + jobPool: "linker" + jobCount: 1 + } + // ... + } + \endcode + The above construct ensures that this specific library is never linked at + the same time as any other binary in the project. + + Job limits set on the command line override those from the settings, which in turn + override the ones defined within a project. Use the \c{--enforce-project-job-limits} + option to give the job limits defined via \c JobLimit items maximum precedence. */ diff --git a/doc/reference/cli/builtin/cli-build.qdoc b/doc/reference/cli/builtin/cli-build.qdoc index 8e4b8ed44..cffb19d49 100644 --- a/doc/reference/cli/builtin/cli-build.qdoc +++ b/doc/reference/cli/builtin/cli-build.qdoc @@ -68,6 +68,7 @@ \target build-force-probe-execution \include cli-options.qdocinc force-probe-execution \include cli-options.qdocinc jobs + \include cli-options.qdocinc job-limits \include cli-options.qdocinc keep-going \include cli-options.qdocinc less-verbose \include cli-options.qdocinc log-level diff --git a/doc/reference/cli/cli-options.qdocinc b/doc/reference/cli/cli-options.qdocinc index 189a3526a..e6d909963 100644 --- a/doc/reference/cli/cli-options.qdocinc +++ b/doc/reference/cli/cli-options.qdocinc @@ -249,6 +249,22 @@ //! [jobs] +//! [job-limits] + + \section2 \c {--job-limits :[,:...]} + + Sets pool-specific job limits. See \l{job-pool-howto}{here} for more information on + job pools. + + \section2 \c {--enforce-project-job-limits} + + Normally, job limits defined in project files via the \l JobLimit item get overridden + by those set on the command line. If this option is passed, they get maximum priority + instead. Use it if there are product-specific limits that make more sense for + that part of the code base than the generic ones you'd like to apply globally. + +//! [jobs] + //! [keep-going] \section2 \c --keep-going|-k diff --git a/doc/reference/commands.qdoc b/doc/reference/commands.qdoc index 8f0392c32..eb93e8f7a 100644 --- a/doc/reference/commands.qdoc +++ b/doc/reference/commands.qdoc @@ -102,6 +102,13 @@ \li "filegen" indicates that the command creates arbitrary files \endlist All other values are mapped to the default color. + \row + \li \c jobPool + \li string + \li empty + \li Determines which job pool the command will use. An empty + string, which is the default, stands for the global job pool. + See \l{JobLimit}{here} and \l{job-pool-howto}{here} for more information on job pools. \row \li \c silent \li bool diff --git a/doc/reference/items/language/group.qdoc b/doc/reference/items/language/group.qdoc index dae647060..caeffcfaa 100644 --- a/doc/reference/items/language/group.qdoc +++ b/doc/reference/items/language/group.qdoc @@ -28,7 +28,7 @@ /*! \contentspage list-of-language-items.html \previouspage FileTagger - \nextpage Module + \nextpage JobLimit \qmltype Group \inqmlmodule QbsLanguageItems \ingroup list-of-items diff --git a/doc/reference/items/language/joblimit.qdoc b/doc/reference/items/language/joblimit.qdoc new file mode 100644 index 000000000..66697ba3e --- /dev/null +++ b/doc/reference/items/language/joblimit.qdoc @@ -0,0 +1,107 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:FDL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Free Documentation License Usage +** Alternatively, this file may be used under the terms of the GNU Free +** Documentation License version 1.3 as published by the Free Software +** Foundation and appearing in the file included in the packaging of +** this file. Please review the following information to ensure +** the GNU Free Documentation License version 1.3 requirements +** will be met: https://www.gnu.org/licenses/fdl-1.3.html. +** $QT_END_LICENSE$ +** +****************************************************************************/ +/*! + \contentspage list-of-language-items.html + \previouspage Group + \nextpage Module + \qmltype JobLimit + \inqmlmodule QbsLanguageItems + \ingroup list-of-items + \keyword QML.JobLimit + + \brief Restricts concurrent execution of jobs in a given pool. + + In addition to the global limit on concurrently running commands, a project might + want to restrict concurrent execution of certain types of commands even further, + for instance because they are not well-suited to share certain types of resources. + + In the following example, we define a rule that runs a tool of which at most one + instance can be running for the same project at any given time: + \code + Rule { + // ... + prepare: { + var cmd = new Command("my-exclusive-tool", [project.buildDirectory]); + cmd.description = "running the exclusive tool"; + cmd.jobPool = "exclusive_tool"; + return cmd; + } + } + JobLimit { + jobPool: "exclusive_tool" + jobCount: 1 + } + \endcode + + \c JobLimit items can appear inside \l Product, \l Project and \l Module items. + In the case of collisions, that is, items matching the same job pool but setting + different values, the ones defined inside products have the highest precedence, + and the ones inside modules have the lowest. Items defined in sub-projects have + higher precedence than those defined in parent projects. For items with the same + precedence level, the most restrictive one is chosen, that is, the one with the + lowest job number greater than zero. + + \see {How do I limit the number of concurrent jobs for the linker only?} +*/ + +/*! + \qmlproperty bool JobLimit::condition + + Determines whether the job limit is active. + + If this property is set to \c false, the job limit is ignored. + + \defaultvalue \c true +*/ + +/*! + \qmlproperty string JobLimit::jobCount + + The maximum number of commands in the given \l{jobPool}{job pool} that can run + concurrently. + + A value of zero means "unlimited", negative values are not allowed. + + \note The global job limit always applies: For instance, if you set this + property to 100 for some job pool, and "-j 8" was given on the + command line, then no more than eight instances of commands from + the respective job pool will run at any time. + + This property must always be set. + + \nodefaultvalue +*/ + +/*! + \qmlproperty string JobLimit::jobPool + + The job pool to which apply the limit. + + This property must always be set to a non-empty value. + + \nodefaultvalue +*/ diff --git a/doc/reference/items/language/module.qdoc b/doc/reference/items/language/module.qdoc index bfaedacc9..e5472983f 100644 --- a/doc/reference/items/language/module.qdoc +++ b/doc/reference/items/language/module.qdoc @@ -26,7 +26,7 @@ ****************************************************************************/ /*! \contentspage list-of-language-items.html - \previouspage Group + \previouspage JobLimit \nextpage Parameter \qmltype Module \inqmlmodule QbsLanguageItems diff --git a/doc/reference/modules/cpp-module.qdoc b/doc/reference/modules/cpp-module.qdoc index 0818af28d..386ac7553 100644 --- a/doc/reference/modules/cpp-module.qdoc +++ b/doc/reference/modules/cpp-module.qdoc @@ -217,6 +217,31 @@ This file tag only has an effect with GCC-like toolchains. The linker needs to be \c{ld}-compatible. \endtable + + \section2 Relevant Job Pools + \target cpp-job-pools + + \table + \header + \li Pool + \li Since + \li Description + \row + \li \c{"assembler"} + \li 1.13 + \li The job pool used by rules that run the toolchain's assembler. This is only + relevant for direct invocations of the assembler binary, not for running it + indirectly via the compiler. + \row + \li \c{"compiler"} + \li 1.13 + \li The job pool used by rules that run a compiler. All language variants use + the same pool. + \row + \li \c{"linker"} + \li 1.13 + \li The job pool used by rules that run a linker. + \endtable */ /*! diff --git a/share/qbs/modules/cpp/GenericGCC.qbs b/share/qbs/modules/cpp/GenericGCC.qbs index 423d964d5..79575ca2b 100644 --- a/share/qbs/modules/cpp/GenericGCC.qbs +++ b/share/qbs/modules/cpp/GenericGCC.qbs @@ -494,6 +494,7 @@ CppModule { var cmd = new Command(product.cpp.archiverPath, args); cmd.description = 'creating ' + output.fileName; cmd.highlight = 'linker' + cmd.jobPool = "linker"; cmd.responseFileUsagePrefix = '@'; return cmd; } diff --git a/share/qbs/modules/cpp/gcc.js b/share/qbs/modules/cpp/gcc.js index 6e8622b93..ff7867b25 100644 --- a/share/qbs/modules/cpp/gcc.js +++ b/share/qbs/modules/cpp/gcc.js @@ -984,6 +984,7 @@ function prepareAssembler(project, product, inputs, outputs, input, output) { var cmd = new Command(assemblerPath, args); cmd.description = "assembling " + input.fileName; cmd.highlight = "compiler"; + cmd.jobPool = "assembler"; return cmd; } @@ -1055,6 +1056,7 @@ function prepareCompiler(project, product, inputs, outputs, input, output, expli if (pchOutput) cmd.description += ' (' + compilerInfo.tag + ')'; cmd.highlight = "compiler"; + cmd.jobPool = "compiler"; cmd.relevantEnvironmentVariables = compilerEnvVars(input, compilerInfo); cmd.responseFileArgumentIndex = wrapperArgsLength; cmd.responseFileUsagePrefix = '@'; @@ -1274,6 +1276,7 @@ function prepareLinker(project, product, inputs, outputs, input, output) { cmd = new Command(linkerPath, args); cmd.description = 'linking ' + primaryOutput.fileName; cmd.highlight = 'linker'; + cmd.jobPool = "linker"; cmd.relevantEnvironmentVariables = linkerEnvVars(product, inputs); cmd.responseFileArgumentIndex = responseFileArgumentIndex; cmd.responseFileUsagePrefix = useQnxResponseFileHack ? "-Wl,@" : "@"; diff --git a/share/qbs/modules/cpp/msvc.js b/share/qbs/modules/cpp/msvc.js index d6df99dc2..196e5ae43 100644 --- a/share/qbs/modules/cpp/msvc.js +++ b/share/qbs/modules/cpp/msvc.js @@ -246,6 +246,7 @@ function prepareCompiler(project, product, inputs, outputs, input, output, expli if (pchOutput) cmd.description += ' (' + tag + ')'; cmd.highlight = "compiler"; + cmd.jobPool = "compiler"; cmd.workingDirectory = product.buildDirectory; cmd.responseFileUsagePrefix = '@'; // cl.exe outputs the cpp file name. We filter that out. @@ -469,6 +470,7 @@ function prepareLinker(project, product, inputs, outputs, input, output) { var cmd = new Command(linkerPath, args) cmd.description = 'linking ' + primaryOutput.fileName; cmd.highlight = 'linker'; + cmd.jobPool = "linker"; cmd.relevantEnvironmentVariables = ["LINK", "_LINK_", "LIB", "TMP"]; cmd.workingDirectory = FileInfo.path(primaryOutput.filePath) cmd.responseFileUsagePrefix = '@'; diff --git a/share/qbs/modules/cpp/windows-msvc.qbs b/share/qbs/modules/cpp/windows-msvc.qbs index 18bc9705c..b22984a6e 100644 --- a/share/qbs/modules/cpp/windows-msvc.qbs +++ b/share/qbs/modules/cpp/windows-msvc.qbs @@ -287,6 +287,7 @@ CppModule { var cmd = new Command("lib.exe", args); cmd.description = 'creating ' + lib.fileName; cmd.highlight = 'linker'; + cmd.jobPool = "linker"; cmd.workingDirectory = FileInfo.path(lib.filePath) cmd.responseFileUsagePrefix = '@'; return cmd; @@ -339,6 +340,7 @@ CppModule { var cmd = new Command('rc', args); cmd.description = 'compiling ' + input.fileName; cmd.highlight = 'compiler'; + cmd.jobPool = "compiler"; if (!hasNoLogo) { // Remove the first two lines of stdout. That's the logo. @@ -375,6 +377,7 @@ CppModule { ModUtils.moduleProperty(input, 'flags', 'asm')); var cmd = new Command(product.cpp.assemblerPath, args); cmd.description = "assembling " + input.fileName; + cmd.jobPool = "assembler"; cmd.inputFileName = input.fileName; cmd.stdoutFilterFunction = function(output) { var lines = output.split("\r\n").filter(function (s) { diff --git a/src/app/qbs/parser/commandlineoption.cpp b/src/app/qbs/parser/commandlineoption.cpp index c2eea340a..e18658751 100644 --- a/src/app/qbs/parser/commandlineoption.cpp +++ b/src/app/qbs/parser/commandlineoption.cpp @@ -574,6 +574,59 @@ void SettingsDirOption::doParse(const QString &representation, QStringList &inpu m_settingsDir = input.takeFirst(); } +QString JobLimitsOption::description(CommandType command) const +{ + Q_UNUSED(command); + return Tr::tr("%1 :[,:...]\n" + "\tSet pool-specific job limits.\n").arg(longRepresentation()); +} + +QString JobLimitsOption::longRepresentation() const +{ + return QLatin1String("--job-limits"); +} + +void JobLimitsOption::doParse(const QString &representation, QStringList &input) +{ + if (input.empty()) { + throw ErrorInfo(Tr::tr("Invalid use of option '%1: Argument expected.\n" + "Usage: %2").arg(representation, description(command()))); + } + const QString jobLimitsSpec = input.takeFirst(); + const QStringList jobLimitStrings = jobLimitsSpec.split(QLatin1Char(',')); + for (const QString &jobLimitString : jobLimitStrings) { + const int sepIndex = jobLimitString.indexOf(QLatin1Char(':')); + if (sepIndex <= 0 || sepIndex == jobLimitString.size() - 1) { + throw ErrorInfo(Tr::tr("Invalid use of option '%1: " + "Invalid job limits specification '%2'.\n" + "Usage: %3").arg(representation, jobLimitsSpec, + description(command()))); + } + const QString pool = jobLimitString.left(sepIndex); + const QString limitString = jobLimitString.mid(sepIndex + 1); + bool isValidNumber; + const int limit = limitString.toInt(&isValidNumber); + if (!isValidNumber) { + throw ErrorInfo(Tr::tr("Invalid use of option '%1: '%2' is not a number.\n" + "Usage: %3").arg(representation, limitString, + description(command()))); + } + m_jobLimits.setJobLimit(pool, limit); + } +} + +QString RespectProjectJobLimitsOption::description(CommandType command) const +{ + Q_UNUSED(command); + return Tr::tr("%1\n\tGive maximum priority to job limits defined inside the project.\n") + .arg(longRepresentation()); +} + +QString RespectProjectJobLimitsOption::longRepresentation() const +{ + return QLatin1String("--enforce-project-job-limits"); +} + CommandEchoModeOption::CommandEchoModeOption() { } diff --git a/src/app/qbs/parser/commandlineoption.h b/src/app/qbs/parser/commandlineoption.h index c645c533b..d57ec76b7 100644 --- a/src/app/qbs/parser/commandlineoption.h +++ b/src/app/qbs/parser/commandlineoption.h @@ -42,6 +42,7 @@ #include "commandtype.h" #include +#include #include @@ -69,6 +70,8 @@ public: LogTimeOptionType, CommandEchoModeOptionType, SettingsDirOptionType, + JobLimitsOptionType, + RespectProjectJobLimitsOptionType, GeneratorOptionType, WaitLockOptionType, RunEnvConfigOptionType, @@ -380,6 +383,29 @@ private: QString m_settingsDir; }; +class JobLimitsOption : public CommandLineOption +{ +public: + JobLimits jobLimits() const { return m_jobLimits; } + + QString description(CommandType command) const override; + QString shortRepresentation() const override { return QString(); } + QString longRepresentation() const override; + +private: + void doParse(const QString &representation, QStringList &input) override; + + JobLimits m_jobLimits; +}; + +class RespectProjectJobLimitsOption : public OnOffOption +{ +public: + QString description(CommandType command) const override; + QString shortRepresentation() const override { return QString(); } + QString longRepresentation() const override; +}; + class WaitLockOption : public OnOffOption { public: diff --git a/src/app/qbs/parser/commandlineoptionpool.cpp b/src/app/qbs/parser/commandlineoptionpool.cpp index 8850fad87..9964f051a 100644 --- a/src/app/qbs/parser/commandlineoptionpool.cpp +++ b/src/app/qbs/parser/commandlineoptionpool.cpp @@ -116,6 +116,12 @@ CommandLineOption *CommandLineOptionPool::getOption(CommandLineOption::Type type case CommandLineOption::SettingsDirOptionType: option = new SettingsDirOption; break; + case CommandLineOption::JobLimitsOptionType: + option = new JobLimitsOption; + break; + case CommandLineOption::RespectProjectJobLimitsOptionType: + option = new RespectProjectJobLimitsOption; + break; case CommandLineOption::GeneratorOptionType: option = new GeneratorOption; break; @@ -246,6 +252,17 @@ SettingsDirOption *CommandLineOptionPool::settingsDirOption() const return static_cast(getOption(CommandLineOption::SettingsDirOptionType)); } +JobLimitsOption *CommandLineOptionPool::jobLimitsOption() const +{ + return static_cast(getOption(CommandLineOption::JobLimitsOptionType)); +} + +RespectProjectJobLimitsOption *CommandLineOptionPool::respectProjectJobLimitsOption() const +{ + return static_cast( + getOption(CommandLineOption::RespectProjectJobLimitsOptionType)); +} + GeneratorOption *CommandLineOptionPool::generatorOption() const { return static_cast(getOption(CommandLineOption::GeneratorOptionType)); diff --git a/src/app/qbs/parser/commandlineoptionpool.h b/src/app/qbs/parser/commandlineoptionpool.h index a4e6c30c1..6a4669165 100644 --- a/src/app/qbs/parser/commandlineoptionpool.h +++ b/src/app/qbs/parser/commandlineoptionpool.h @@ -73,6 +73,8 @@ public: LogTimeOption *logTimeOption() const; CommandEchoModeOption *commandEchoModeOption() const; SettingsDirOption *settingsDirOption() const; + JobLimitsOption *jobLimitsOption() const; + RespectProjectJobLimitsOption *respectProjectJobLimitsOption() const; GeneratorOption *generatorOption() const; WaitLockOption *waitLockOption() const; RunEnvConfigOption *runEnvConfigOption() const; diff --git a/src/app/qbs/parser/commandlineparser.cpp b/src/app/qbs/parser/commandlineparser.cpp index 92c43c8c6..c2e265336 100644 --- a/src/app/qbs/parser/commandlineparser.cpp +++ b/src/app/qbs/parser/commandlineparser.cpp @@ -462,6 +462,10 @@ void CommandLineParser::CommandLineParserPrivate::setupBuildOptions() buildOptions.setEchoMode(echoMode()); buildOptions.setInstall(!optionPool.noInstallOption()->enabled()); buildOptions.setRemoveExistingInstallation(optionPool.removeFirstoption()->enabled()); + buildOptions.setJobLimits(optionPool.jobLimitsOption()->jobLimits()); + buildOptions.setProjectJobLimitsTakePrecedence( + optionPool.respectProjectJobLimitsOption()->enabled()); + buildOptions.setSettingsDirectory(settingsDir()); } void CommandLineParser::CommandLineParserPrivate::setupBuildConfigurations() diff --git a/src/app/qbs/parser/parsercommand.cpp b/src/app/qbs/parser/parsercommand.cpp index b85e98540..ff331fd50 100644 --- a/src/app/qbs/parser/parsercommand.cpp +++ b/src/app/qbs/parser/parsercommand.cpp @@ -287,6 +287,8 @@ static QList buildOptions() << CommandLineOption::CommandEchoModeOptionType << CommandLineOption::NoInstallOptionType << CommandLineOption::RemoveFirstOptionType + << CommandLineOption::JobLimitsOptionType + << CommandLineOption::RespectProjectJobLimitsOptionType << CommandLineOption::WaitLockOptionType; } diff --git a/src/lib/corelib/buildgraph/executor.cpp b/src/lib/corelib/buildgraph/executor.cpp index f03529eec..368fa6ccb 100644 --- a/src/lib/corelib/buildgraph/executor.cpp +++ b/src/lib/corelib/buildgraph/executor.cpp @@ -61,10 +61,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include @@ -230,6 +232,9 @@ void Executor::doBuild() m_tagsNeededForFilesToConsider.clear(); m_productsOfFilesToConsider.clear(); m_artifactsRemovedFromDisk.clear(); + m_jobCountPerPool.clear(); + + setupJobLimits(); // TODO: The "filesToConsider" thing is badly designed; we should know exactly which artifact // it is. Remove this from the BuildOptions class and introduce Project::buildSomeFiles() @@ -346,6 +351,7 @@ void Executor::updateLeaves(BuildGraphNode *node, NodeSet &seenNodes) bool Executor::scheduleJobs() { QBS_CHECK(m_state == ExecutorRunning); + std::vector delayedLeaves; while (!m_leaves.empty() && !m_availableJobs.empty()) { BuildGraphNode * const nodeToBuild = m_leaves.top(); m_leaves.pop(); @@ -355,9 +361,17 @@ bool Executor::scheduleJobs() QBS_ASSERT(!"untouched node in leaves list", qDebug("%s", qPrintable(nodeToBuild->toString()))); break; - case BuildGraphNode::Buildable: - // This is the only state in which we want to build a node. - nodeToBuild->accept(this); + case BuildGraphNode::Buildable: // This is the only state in which we want to build a node. + // TODO: It's a bit annoying that we have to check this here already, when we + // don't know whether the transformer needs to run at all. Investigate + // moving the whole job allocation logic to runTransformer(). + if (schedulingBlockedByJobLimit(nodeToBuild)) { + qCDebug(lcExec).noquote() << "node delayed due to occupied job pool:" + << nodeToBuild->toString(); + delayedLeaves.push_back(nodeToBuild); + } else { + nodeToBuild->accept(this); + } break; case BuildGraphNode::Building: qCDebug(lcExec).noquote() << nodeToBuild->toString(); @@ -369,9 +383,50 @@ bool Executor::scheduleJobs() break; } } + for (BuildGraphNode * const delayedLeaf : delayedLeaves) + m_leaves.push(delayedLeaf); return !m_leaves.empty() || !m_processingJobs.empty(); } +bool Executor::schedulingBlockedByJobLimit(const BuildGraphNode *node) +{ + if (node->type() != BuildGraphNode::ArtifactNodeType) + return false; + const Artifact * const artifact = static_cast(node); + if (artifact->artifactType == Artifact::SourceFile) + return false; + + const Transformer * const transformer = artifact->transformer.get(); + for (const QString &jobPool : transformer->jobPools()) { + const int currentJobCount = m_jobCountPerPool[jobPool]; + if (currentJobCount == 0) + continue; + const auto jobLimitIsExceeded = [currentJobCount, jobPool, this](const Transformer *t) { + const int maxJobCount = m_jobLimitsPerProduct.at(t->product().get()) + .getLimit(jobPool); + return maxJobCount > 0 && currentJobCount >= maxJobCount; + }; + + // Different products can set different limits. The effective limit is the minimum of what + // is set in this transformer's product and in the products of all currently + // running transformers. + if (jobLimitIsExceeded(transformer)) + return true; + for (const ExecutorJob * const runningJob : m_processingJobs.keys()) { + if (!runningJob->jobPools().contains(jobPool)) + continue; + const Transformer * const runningTransformer = runningJob->transformer(); + if (!runningTransformer) + continue; // This can happen if the ExecutorJob has already finished. + if (runningTransformer->product() == transformer->product()) + continue; // We have already checked this product's job limit. + if (jobLimitIsExceeded(runningTransformer)) + return true; + } + } + return false; +} + bool Executor::isUpToDate(Artifact *artifact) const { QBS_CHECK(artifact->artifactType == Artifact::Generated); @@ -503,6 +558,7 @@ void Executor::finishJob(ExecutorJob *job, bool success) const TransformerPtr transformer = it.value(); m_processingJobs.erase(it); m_availableJobs.push_back(job); + updateJobCounts(transformer.get(), -1); if (success) { m_project->buildData->setDirty(); for (Artifact * const artifact : qAsConst(transformer->outputs)) { @@ -630,6 +686,30 @@ bool Executor::transformerHasMatchingInputFiles(const TransformerConstPtr &trans return false; } +void Executor::setupJobLimits() +{ + Settings settings(m_buildOptions.settingsDirectory()); + for (const ResolvedProductConstPtr &p : m_productsToBuild) { + const Preferences prefs(&settings, p->profile()); + const JobLimits &jobLimitsFromSettings = prefs.jobLimits(); + JobLimits effectiveJobLimits; + if (m_buildOptions.projectJobLimitsTakePrecedence()) { + effectiveJobLimits.update(jobLimitsFromSettings).update(m_buildOptions.jobLimits()) + .update(p->jobLimits); + } else { + effectiveJobLimits.update(p->jobLimits).update(jobLimitsFromSettings) + .update(m_buildOptions.jobLimits()); + } + m_jobLimitsPerProduct.insert(std::make_pair(p.get(), effectiveJobLimits)); + } +} + +void Executor::updateJobCounts(const Transformer *transformer, int diff) +{ + for (const QString &jobPool : transformer->jobPools()) + m_jobCountPerPool[jobPool] += diff; +} + void Executor::cancelJobs() { if (m_state == ExecutorCanceling) @@ -923,6 +1003,7 @@ void Executor::runTransformer(const TransformerPtr &transformer) for (Artifact * const artifact : qAsConst(transformer->outputs)) artifact->buildState = BuildGraphNode::Building; m_processingJobs.insert(job, transformer); + updateJobCounts(transformer.get(), 1); job->run(transformer.get()); } diff --git a/src/lib/corelib/buildgraph/executor.h b/src/lib/corelib/buildgraph/executor.h index 7225c0ace..4b4951e1e 100644 --- a/src/lib/corelib/buildgraph/executor.h +++ b/src/lib/corelib/buildgraph/executor.h @@ -154,6 +154,10 @@ private: bool artifactHasMatchingOutputTags(const Artifact *artifact) const; bool transformerHasMatchingInputFiles(const TransformerConstPtr &transformer) const; + void setupJobLimits(); + void updateJobCounts(const Transformer *transformer, int diff); + bool schedulingBlockedByJobLimit(const BuildGraphNode *node); + typedef QHash JobMap; JobMap m_processingJobs; @@ -169,6 +173,8 @@ private: std::vector m_allProducts; std::unordered_map m_productsByName; std::unordered_map m_projectsByName; + std::unordered_map m_jobCountPerPool; + std::unordered_map m_jobLimitsPerProduct; NodeSet m_roots; Leaves m_leaves; InputArtifactScannerContext *m_inputArtifactScanContext; diff --git a/src/lib/corelib/buildgraph/executorjob.cpp b/src/lib/corelib/buildgraph/executorjob.cpp index e90b498f6..79f17377d 100644 --- a/src/lib/corelib/buildgraph/executorjob.cpp +++ b/src/lib/corelib/buildgraph/executorjob.cpp @@ -113,6 +113,7 @@ void ExecutorJob::run(Transformer *t) m_processCommandExecutor->setProcessEnvironment( (*t->outputs.cbegin())->product->buildEnvironment); m_transformer = t; + m_jobPools = t->jobPools(); runNextCommand(); } @@ -171,6 +172,7 @@ void ExecutorJob::setFinished() void ExecutorJob::reset() { m_transformer = nullptr; + m_jobPools.clear(); m_currentCommandExecutor = nullptr; m_currentCommandIdx = -1; m_error.clear(); diff --git a/src/lib/corelib/buildgraph/executorjob.h b/src/lib/corelib/buildgraph/executorjob.h index b1701a11d..e28d42f7f 100644 --- a/src/lib/corelib/buildgraph/executorjob.h +++ b/src/lib/corelib/buildgraph/executorjob.h @@ -43,8 +43,10 @@ #include #include #include +#include #include +#include namespace qbs { class CodeLocation; @@ -71,6 +73,8 @@ public: void setEchoMode(CommandEchoMode echoMode); void run(Transformer *t); void cancel(); + const Transformer *transformer() const { return m_transformer; } + Set jobPools() const { return m_jobPools; } signals: void reportCommandDescription(const QString &highlight, const QString &message); @@ -88,6 +92,7 @@ private: ProcessCommandExecutor *m_processCommandExecutor; JsCommandExecutor *m_jsCommandExecutor; Transformer *m_transformer; + Set m_jobPools; int m_currentCommandIdx; ErrorInfo m_error; }; diff --git a/src/lib/corelib/buildgraph/rulecommands.cpp b/src/lib/corelib/buildgraph/rulecommands.cpp index c2aad1ce1..f7620dbf8 100644 --- a/src/lib/corelib/buildgraph/rulecommands.cpp +++ b/src/lib/corelib/buildgraph/rulecommands.cpp @@ -103,6 +103,7 @@ bool AbstractCommand::equals(const AbstractCommand *other) const && m_highlight == other->m_highlight && m_ignoreDryRun == other->m_ignoreDryRun && m_silent == other->m_silent + && m_jobPool == other->m_jobPool && m_properties == other->m_properties; } @@ -113,6 +114,7 @@ void AbstractCommand::fillFromScriptValue(const QScriptValue *scriptValue, const m_highlight = scriptValue->property(highlightProperty()).toString(); m_ignoreDryRun = scriptValue->property(ignoreDryRunProperty()).toBool(); m_silent = scriptValue->property(silentProperty()).toBool(); + m_jobPool = scriptValue->property(StringConstants::jobPoolProperty()).toString(); m_codeLocation = codeLocation; m_predefinedProperties @@ -120,6 +122,7 @@ void AbstractCommand::fillFromScriptValue(const QScriptValue *scriptValue, const << extendedDescriptionProperty() << highlightProperty() << ignoreDryRunProperty() + << StringConstants::jobPoolProperty() << silentProperty(); } diff --git a/src/lib/corelib/buildgraph/rulecommands.h b/src/lib/corelib/buildgraph/rulecommands.h index 7583de2b6..02d1a02a5 100644 --- a/src/lib/corelib/buildgraph/rulecommands.h +++ b/src/lib/corelib/buildgraph/rulecommands.h @@ -81,6 +81,7 @@ public: const QString highlight() const { return m_highlight; } bool ignoreDryRun() const { return m_ignoreDryRun; } bool isSilent() const { return m_silent; } + QString jobPool() const { return m_jobPool; } CodeLocation codeLocation() const { return m_codeLocation; } const QVariantMap &properties() const { return m_properties; } @@ -98,7 +99,8 @@ private: template void serializationOp(PersistentPool &pool) { pool.serializationOp(m_description, m_extendedDescription, m_highlight, - m_ignoreDryRun, m_silent, m_codeLocation, m_properties); + m_ignoreDryRun, m_silent, m_codeLocation, m_jobPool, + m_properties); } QString m_description; @@ -107,6 +109,7 @@ private: bool m_ignoreDryRun; bool m_silent; CodeLocation m_codeLocation; + QString m_jobPool; QVariantMap m_properties; }; diff --git a/src/lib/corelib/buildgraph/transformer.cpp b/src/lib/corelib/buildgraph/transformer.cpp index df371e1fe..4569b9955 100644 --- a/src/lib/corelib/buildgraph/transformer.cpp +++ b/src/lib/corelib/buildgraph/transformer.cpp @@ -305,5 +305,15 @@ void Transformer::rescueChangeTrackingData(const TransformerConstPtr &other) exportedModulesAccessedInCommands = other->exportedModulesAccessedInCommands; } +Set Transformer::jobPools() const +{ + Set pools; + for (const AbstractCommandPtr &c : commands.commands()) { + if (!c->jobPool().isEmpty()) + pools.insert(c->jobPool()); + } + return pools; +} + } // namespace Internal } // namespace qbs diff --git a/src/lib/corelib/buildgraph/transformer.h b/src/lib/corelib/buildgraph/transformer.h index ce66a976e..2f6a8e56d 100644 --- a/src/lib/corelib/buildgraph/transformer.h +++ b/src/lib/corelib/buildgraph/transformer.h @@ -102,6 +102,8 @@ public: const QScriptValueList &args); void rescueChangeTrackingData(const TransformerConstPtr &other); + Set jobPools() const; + template void completeSerializationOp(PersistentPool &pool) { pool.serializationOp(rule, inputs, outputs, explicitlyDependsOn, diff --git a/src/lib/corelib/corelib.qbs b/src/lib/corelib/corelib.qbs index 93509763b..c947cb484 100644 --- a/src/lib/corelib/corelib.qbs +++ b/src/lib/corelib/corelib.qbs @@ -397,6 +397,7 @@ QbsLibrary { "id.cpp", "id.h", "iosutils.h", + "joblimits.cpp", "jsliterals.cpp", "jsliterals.h", "installoptions.cpp", @@ -467,6 +468,7 @@ QbsLibrary { "error.h", "generateoptions.h", "installoptions.h", + "joblimits.h", "preferences.h", "processresult.h", "profile.h", diff --git a/src/lib/corelib/language/builtindeclarations.cpp b/src/lib/corelib/language/builtindeclarations.cpp index 4be405112..4886675d0 100644 --- a/src/lib/corelib/language/builtindeclarations.cpp +++ b/src/lib/corelib/language/builtindeclarations.cpp @@ -67,6 +67,7 @@ BuiltinDeclarations::BuiltinDeclarations() { QLatin1String("Export"), ItemType::Export }, { QLatin1String("FileTagger"), ItemType::FileTagger }, { QLatin1String("Group"), ItemType::Group }, + { QLatin1String("JobLimit"), ItemType::JobLimit }, { QLatin1String("Module"), ItemType::Module }, { QLatin1String("Parameter"), ItemType::Parameter }, { QLatin1String("Parameters"), ItemType::Parameters }, @@ -87,6 +88,7 @@ BuiltinDeclarations::BuiltinDeclarations() addExportItem(); addFileTaggerItem(); addGroupItem(); + addJobLimitItem(); addModuleItem(); addProbeItem(); addProductItem(); @@ -298,6 +300,15 @@ void BuiltinDeclarations::addGroupItem() insert(item); } +void BuiltinDeclarations::addJobLimitItem() +{ + ItemDeclaration item(ItemType::JobLimit); + item << conditionProperty(); + item << PropertyDeclaration(StringConstants::jobPoolProperty(), PropertyDeclaration::String); + item << PropertyDeclaration(StringConstants::jobCountProperty(), PropertyDeclaration::Integer); + insert(item); +} + void BuiltinDeclarations::addModuleItem() { ItemDeclaration item = moduleLikeItem(ItemType::Module); @@ -312,6 +323,7 @@ ItemDeclaration BuiltinDeclarations::moduleLikeItem(ItemType type) << ItemType::Group << ItemType::Depends << ItemType::FileTagger + << ItemType::JobLimit << ItemType::Rule << ItemType::Parameter << ItemType::Probe @@ -360,6 +372,7 @@ void BuiltinDeclarations::addProductItem() << ItemType::Depends << ItemType::Group << ItemType::FileTagger + << ItemType::JobLimit << ItemType::Export << ItemType::Probe << ItemType::Profile @@ -429,6 +442,7 @@ void BuiltinDeclarations::addProjectItem() << ItemType::Profile << ItemType::Probe << ItemType::FileTagger + << ItemType::JobLimit << ItemType::Rule); item << nameProperty(); item << conditionProperty(); diff --git a/src/lib/corelib/language/builtindeclarations.h b/src/lib/corelib/language/builtindeclarations.h index c75475df7..ff16b395a 100644 --- a/src/lib/corelib/language/builtindeclarations.h +++ b/src/lib/corelib/language/builtindeclarations.h @@ -76,6 +76,7 @@ private: void addExportItem(); void addFileTaggerItem(); void addGroupItem(); + void addJobLimitItem(); void addModuleItem(); static ItemDeclaration moduleLikeItem(ItemType type); void addProbeItem(); diff --git a/src/lib/corelib/language/itemtype.h b/src/lib/corelib/language/itemtype.h index c0e76c94b..324a1fb87 100644 --- a/src/lib/corelib/language/itemtype.h +++ b/src/lib/corelib/language/itemtype.h @@ -53,6 +53,7 @@ enum class ItemType { Export, FileTagger, Group, + JobLimit, Module, Parameter, Parameters, diff --git a/src/lib/corelib/language/language.h b/src/lib/corelib/language/language.h index 580b3810c..06a9af54a 100644 --- a/src/lib/corelib/language/language.h +++ b/src/lib/corelib/language/language.h @@ -49,6 +49,7 @@ #include #include #include +#include #include #include #include @@ -571,6 +572,7 @@ public: std::vector dependencies; QHash dependencyParameters; std::vector fileTaggers; + JobLimits jobLimits; std::vector modules; QHash moduleParameters; std::vector scanners; @@ -628,7 +630,8 @@ private: missingSourceFiles, location, productProperties, moduleProperties, rules, dependencies, dependencyParameters, fileTaggers, modules, moduleParameters, scanners, groups, - artifactProperties, probes, exportedModule, buildData); + artifactProperties, probes, exportedModule, buildData, + jobLimits); } QHash m_executablePathCache; diff --git a/src/lib/corelib/language/projectresolver.cpp b/src/lib/corelib/language/projectresolver.cpp index ec29e00c0..27829e378 100644 --- a/src/lib/corelib/language/projectresolver.cpp +++ b/src/lib/corelib/language/projectresolver.cpp @@ -56,6 +56,7 @@ #include #include #include +#include #include #include #include @@ -89,6 +90,7 @@ struct ProjectResolver::ProjectContext ResolvedProjectPtr project; std::vector fileTaggers; std::vector rules; + JobLimits jobLimits; ResolvedModulePtr dummyModule; }; @@ -106,6 +108,7 @@ struct ProjectResolver::ProductContext struct ProjectResolver::ModuleContext { ResolvedModulePtr module; + JobLimits jobLimits; }; class CancelException { }; @@ -333,6 +336,7 @@ void ProjectResolver::resolveProjectFully(Item *item, ProjectResolver::ProjectCo { ItemType::Product, &ProjectResolver::resolveProduct }, { ItemType::Probe, &ProjectResolver::ignoreItem }, { ItemType::FileTagger, &ProjectResolver::resolveFileTagger }, + { ItemType::JobLimit, &ProjectResolver::resolveJobLimit }, { ItemType::Rule, &ProjectResolver::resolveRule }, { ItemType::PropertyOptions, &ProjectResolver::ignoreItem } }; @@ -494,6 +498,7 @@ void ProjectResolver::resolveProductFully(Item *item, ProjectContext *projectCon { ItemType::Depends, &ProjectResolver::ignoreItem }, { ItemType::Rule, &ProjectResolver::resolveRule }, { ItemType::FileTagger, &ProjectResolver::resolveFileTagger }, + { ItemType::JobLimit, &ProjectResolver::resolveJobLimit }, { ItemType::Group, &ProjectResolver::resolveGroup }, { ItemType::Product, &ProjectResolver::resolveShadowProduct }, { ItemType::Export, &ProjectResolver::resolveExport }, @@ -504,6 +509,11 @@ void ProjectResolver::resolveProductFully(Item *item, ProjectContext *projectCon for (Item * const child : qAsConst(subItems)) callItemFunction(mapping, child, projectContext); + for (const ProjectContext *p = projectContext; p; p = p->parentContext) { + JobLimits tempLimits = p->jobLimits; + product->jobLimits = tempLimits.update(product->jobLimits); + } + resolveModules(item, projectContext); for (const FileTag &t : qAsConst(product->fileTags)) @@ -512,12 +522,19 @@ void ProjectResolver::resolveProductFully(Item *item, ProjectContext *projectCon void ProjectResolver::resolveModules(const Item *item, ProjectContext *projectContext) { + JobLimits jobLimits; for (const Item::Module &m : item->modules()) - resolveModule(m.name, m.item, m.isProduct, m.parameters, projectContext); + resolveModule(m.name, m.item, m.isProduct, m.parameters, jobLimits, projectContext); + for (int i = 0; i < jobLimits.count(); ++i) { + const JobLimit &moduleJobLimit = jobLimits.jobLimitAt(i); + if (m_productContext->product->jobLimits.getLimit(moduleJobLimit.pool()) == -1) + m_productContext->product->jobLimits.setJobLimit(moduleJobLimit); + } } void ProjectResolver::resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct, - const QVariantMap ¶meters, ProjectContext *projectContext) + const QVariantMap ¶meters, JobLimits &jobLimits, + ProjectContext *projectContext) { checkCancelation(); if (!item->isPresentModule()) @@ -550,6 +567,7 @@ void ProjectResolver::resolveModule(const QualifiedId &moduleName, Item *item, b { ItemType::Group, &ProjectResolver::ignoreItem }, { ItemType::Rule, &ProjectResolver::resolveRule }, { ItemType::FileTagger, &ProjectResolver::resolveFileTagger }, + { ItemType::JobLimit, &ProjectResolver::resolveJobLimit }, { ItemType::Scanner, &ProjectResolver::resolveScanner }, { ItemType::PropertyOptions, &ProjectResolver::ignoreItem }, { ItemType::Depends, &ProjectResolver::ignoreItem }, @@ -559,6 +577,12 @@ void ProjectResolver::resolveModule(const QualifiedId &moduleName, Item *item, b }; for (Item *child : item->children()) callItemFunction(mapping, child, projectContext); + for (int i = 0; i < moduleContext.jobLimits.count(); ++i) { + const JobLimit &newJobLimit = moduleContext.jobLimits.jobLimitAt(i); + const int oldLimit = jobLimits.getLimit(newJobLimit.pool()); + if (oldLimit == -1 || oldLimit > newJobLimit.limit()) + jobLimits.setJobLimit(newJobLimit); + } m_moduleContext = oldModuleContext; } @@ -1314,6 +1338,35 @@ void ProjectResolver::resolveFileTagger(Item *item, ProjectContext *projectConte fileTaggers.push_back(FileTagger::create(patterns, fileTags, priority)); } +void ProjectResolver::resolveJobLimit(Item *item, ProjectResolver::ProjectContext *projectContext) +{ + if (!m_evaluator->boolValue(item, StringConstants::conditionProperty())) + return; + const QString jobPool = m_evaluator->stringValue(item, StringConstants::jobPoolProperty()); + if (jobPool.isEmpty()) + throw ErrorInfo(Tr::tr("A JobLimit item needs to have a non-empty '%1' property.") + .arg(StringConstants::jobPoolProperty()), item->location()); + bool jobCountWasSet; + const int jobCount = m_evaluator->intValue(item, StringConstants::jobCountProperty(), -1, + &jobCountWasSet); + if (!jobCountWasSet) { + throw ErrorInfo(Tr::tr("A JobLimit item needs to have a '%1' property.") + .arg(StringConstants::jobCountProperty()), item->location()); + } + if (jobCount < 0) { + throw ErrorInfo(Tr::tr("A JobLimit item must have a non-negative '%1' property.") + .arg(StringConstants::jobCountProperty()), item->location()); + } + JobLimits &jobLimits = m_moduleContext + ? m_moduleContext->jobLimits + : m_productContext ? m_productContext->product->jobLimits + : projectContext->jobLimits; + JobLimit jobLimit(jobPool, jobCount); + const int oldLimit = jobLimits.getLimit(jobPool); + if (oldLimit == -1 || oldLimit > jobCount) + jobLimits.setJobLimit(jobLimit); +} + void ProjectResolver::resolveScanner(Item *item, ProjectResolver::ProjectContext *projectContext) { checkCancelation(); diff --git a/src/lib/corelib/language/projectresolver.h b/src/lib/corelib/language/projectresolver.h index 660bf6259..63a083fef 100644 --- a/src/lib/corelib/language/projectresolver.h +++ b/src/lib/corelib/language/projectresolver.h @@ -56,6 +56,7 @@ #include namespace qbs { +class JobLimits; namespace Internal { class Evaluator; @@ -101,7 +102,8 @@ private: void resolveProductFully(Item *item, ProjectContext *projectContext); void resolveModules(const Item *item, ProjectContext *projectContext); void resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct, - const QVariantMap ¶meters, ProjectContext *projectContext); + const QVariantMap ¶meters, JobLimits &jobLimits, + ProjectContext *projectContext); void gatherProductTypes(ResolvedProduct *product, Item *item); QVariantMap resolveAdditionalModuleProperties(const Item *group, const QVariantMap ¤tValues); @@ -117,6 +119,7 @@ private: const QStringList &namePrefix, QualifiedIdSet *seenBindings); void resolveFileTagger(Item *item, ProjectContext *projectContext); + void resolveJobLimit(Item *item, ProjectContext *projectContext); void resolveScanner(Item *item, ProjectContext *projectContext); void resolveProductDependencies(const ProjectContext &projectContext); void postProcess(const ResolvedProductPtr &product, ProjectContext *projectContext) const; diff --git a/src/lib/corelib/tools/buildoptions.cpp b/src/lib/corelib/tools/buildoptions.cpp index ac8e39867..5507e0842 100644 --- a/src/lib/corelib/tools/buildoptions.cpp +++ b/src/lib/corelib/tools/buildoptions.cpp @@ -58,6 +58,8 @@ public: QStringList changedFiles; QStringList filesToConsider; QStringList activeFileTags; + JobLimits jobLimits; + QString settingsDir; int maxJobCount; bool dryRun; bool keepGoing; @@ -68,6 +70,7 @@ public: bool install; bool removeExistingInstallation; bool onlyExecuteRules; + bool jobLimitsFromProjectTakePrecedence = false; }; } // namespace Internal @@ -188,6 +191,44 @@ void BuildOptions::setMaxJobCount(int jobCount) d->maxJobCount = jobCount; } +/*! + * \brief The base directory for qbs settings. + * This value is used to locate profiles and preferences. + */ +QString BuildOptions::settingsDirectory() const +{ + return d->settingsDir; +} + +/*! + * \brief Sets the base directory for qbs settings. + * \param settingsBaseDir Will be used to locate profiles and preferences. + */ +void BuildOptions::setSettingsDirectory(const QString &settingsBaseDir) +{ + d->settingsDir = settingsBaseDir; +} + +JobLimits BuildOptions::jobLimits() const +{ + return d->jobLimits; +} + +void BuildOptions::setJobLimits(const JobLimits &jobLimits) +{ + d->jobLimits = jobLimits; +} + +bool BuildOptions::projectJobLimitsTakePrecedence() const +{ + return d->jobLimitsFromProjectTakePrecedence; +} + +void BuildOptions::setProjectJobLimitsTakePrecedence(bool toggle) +{ + d->jobLimitsFromProjectTakePrecedence = toggle; +} + /*! * \brief Returns true iff qbs will not actually execute any commands, but just show what * would happen. diff --git a/src/lib/corelib/tools/buildoptions.h b/src/lib/corelib/tools/buildoptions.h index 630a6aa22..cea89d0ea 100644 --- a/src/lib/corelib/tools/buildoptions.h +++ b/src/lib/corelib/tools/buildoptions.h @@ -42,6 +42,7 @@ #include "qbs_export.h" #include "commandechomode.h" +#include "joblimits.h" #include @@ -73,6 +74,15 @@ public: int maxJobCount() const; void setMaxJobCount(int jobCount); + QString settingsDirectory() const; + void setSettingsDirectory(const QString &settingsBaseDir); + + JobLimits jobLimits() const; + void setJobLimits(const JobLimits &jobLimits); + + bool projectJobLimitsTakePrecedence() const; + void setProjectJobLimitsTakePrecedence(bool toggle); + bool dryRun() const; void setDryRun(bool dryRun); diff --git a/src/lib/corelib/tools/joblimits.cpp b/src/lib/corelib/tools/joblimits.cpp new file mode 100644 index 000000000..3b1fde83d --- /dev/null +++ b/src/lib/corelib/tools/joblimits.cpp @@ -0,0 +1,185 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "joblimits.h" + +#include + +#include +#include + +namespace qbs { +namespace Internal { + +static int transformLimit(int limitFromUser) +{ + return limitFromUser == 0 + ? std::numeric_limits::max() + : limitFromUser < -1 ? -1 + : limitFromUser; +} + +class JobLimitPrivate : public QSharedData +{ +public: + JobLimitPrivate(const QString &pool, int limit) + : jobLimit(std::make_pair(pool, transformLimit(limit))) + { + } + template void serializationOp(PersistentPool &pool) + { + pool.serializationOp(jobLimit); + } + std::pair jobLimit; +}; + +class JobLimitsPrivate : public QSharedData +{ +public: + template void serializationOp(PersistentPool &pool) + { + pool.serializationOp(jobLimits); + } + std::vector jobLimits; +}; + +} // namespace Internal + +JobLimit::JobLimit() : JobLimit(QString(), -1) +{ +} +JobLimit::JobLimit(const QString &pool, int limit) : d(new Internal::JobLimitPrivate(pool, limit)) +{ +} +JobLimit::JobLimit(const JobLimit &other) : d(other.d) { } +JobLimit &JobLimit::operator=(const JobLimit &other) +{ + d = other.d; + return *this; +} +JobLimit::~JobLimit() {} +QString JobLimit::pool() const { return d->jobLimit.first; } +int JobLimit::limit() const { return d->jobLimit.second; } + +void JobLimit::load(Internal::PersistentPool &pool) +{ + d->serializationOp(pool); +} + +void JobLimit::store(Internal::PersistentPool &pool) +{ + d->serializationOp(pool); +} + +JobLimits::JobLimits() : d(new Internal::JobLimitsPrivate) { } +JobLimits::JobLimits(const JobLimits &other) : d(other.d) { } +JobLimits &JobLimits::operator=(const JobLimits &other) +{ + d = other.d; + return *this; +} +JobLimits::~JobLimits() {} + +void JobLimits::setJobLimit(const JobLimit &limit) +{ + for (std::size_t i = 0; i < d->jobLimits.size(); ++i) { + JobLimit ¤tLimit = d->jobLimits.at(i); + if (currentLimit.pool() == limit.pool()) { + if (currentLimit.limit() != limit.limit()) + currentLimit = limit; + return; + } + } + d->jobLimits.push_back(limit); +} + +void JobLimits::setJobLimit(const QString &pool, int limit) +{ + setJobLimit(JobLimit(pool, limit)); +} + +int JobLimits::getLimit(const QString &pool) const +{ + for (const JobLimit &l : d->jobLimits) { + if (l.pool() == pool) + return l.limit(); + } + return -1; +} + +bool JobLimits::isEmpty() const +{ + return d->jobLimits.empty(); +} + +int JobLimits::count() const +{ + return d->jobLimits.size(); +} + +JobLimit JobLimits::jobLimitAt(int i) const +{ + return d->jobLimits.at(i); +} + +JobLimits &JobLimits::update(const JobLimits &other) +{ + if (isEmpty()) { + *this = other; + } else { + for (int i = 0; i < other.count(); ++i) { + const JobLimit &l = other.jobLimitAt(i); + if (l.limit() != -1) + setJobLimit(l); + } + } + return *this; +} + +void JobLimits::load(Internal::PersistentPool &pool) +{ + d->serializationOp(pool); +} + +void JobLimits::store(Internal::PersistentPool &pool) +{ + d->serializationOp(pool); +} + +} // namespace qbs diff --git a/src/lib/corelib/tools/joblimits.h b/src/lib/corelib/tools/joblimits.h new file mode 100644 index 000000000..de95f5513 --- /dev/null +++ b/src/lib/corelib/tools/joblimits.h @@ -0,0 +1,102 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#ifndef QBS_JOB_LIMITS_H +#define QBS_JOB_LIMITS_H + +#include "qbs_export.h" + +#include + +QT_BEGIN_NAMESPACE +class QString; +QT_END_NAMESPACE + +namespace qbs { +namespace Internal { +class JobLimitPrivate; +class JobLimitsPrivate; +class PersistentPool; +} + +class QBS_EXPORT JobLimit +{ +public: + JobLimit(); + JobLimit(const QString &pool, int limit); + JobLimit(const JobLimit &other); + JobLimit &operator=(const JobLimit &other); + ~JobLimit(); + + QString pool() const; + int limit() const; + + void load(Internal::PersistentPool &pool); + void store(Internal::PersistentPool &pool); +private: + QSharedDataPointer d; +}; + +class QBS_EXPORT JobLimits +{ +public: + JobLimits(); + JobLimits(const JobLimits &other); + JobLimits &operator=(const JobLimits &other); + ~JobLimits(); + + void setJobLimit(const JobLimit &limit); + void setJobLimit(const QString &pool, int limit); + int getLimit(const QString &pool) const; + bool hasLimit(const QString &pool) const { return getLimit(pool) != -1; } + bool isEmpty() const; + + int count() const; + JobLimit jobLimitAt(int i) const; + + JobLimits &update(const JobLimits &other); + + void load(Internal::PersistentPool &pool); + void store(Internal::PersistentPool &pool); +private: + QSharedDataPointer d; +}; + +} // namespace qbs + +#endif // include guard diff --git a/src/lib/corelib/tools/persistence.cpp b/src/lib/corelib/tools/persistence.cpp index 53c9dfac4..216dfad24 100644 --- a/src/lib/corelib/tools/persistence.cpp +++ b/src/lib/corelib/tools/persistence.cpp @@ -48,7 +48,7 @@ namespace qbs { namespace Internal { -static const char QBS_PERSISTENCE_MAGIC[] = "QBSPERSISTENCE-122"; +static const char QBS_PERSISTENCE_MAGIC[] = "QBSPERSISTENCE-123"; NoBuildGraphError::NoBuildGraphError(const QString &filePath) : ErrorInfo(Tr::tr("Build graph not found for configuration '%1'. Expected location was '%2'.") diff --git a/src/lib/corelib/tools/preferences.cpp b/src/lib/corelib/tools/preferences.cpp index 66803a0f5..12af4e9c7 100644 --- a/src/lib/corelib/tools/preferences.cpp +++ b/src/lib/corelib/tools/preferences.cpp @@ -124,6 +124,32 @@ QStringList Preferences::pluginPaths(const QString &baseDir) const return pathList(QLatin1String("pluginsPath"), baseDir + QLatin1String("/qbs/plugins")); } +/*! + * \brief Returns the per-pool job limits. + */ +JobLimits Preferences::jobLimits() const +{ + const QString prefix = QLatin1String("preferences.jobLimit"); + JobLimits limits; + for (const QString &key : m_settings->allKeysWithPrefix(prefix, Settings::allScopes())) { + limits.setJobLimit(key, m_settings->value(prefix + QLatin1Char('.') + key, + Settings::allScopes()).toInt()); + } + const QString fullPrefix = prefix + QLatin1Char('.'); + if (!m_profile.isEmpty()) { + Profile p(m_profile, m_settings, m_profileContents); + for (const QString &key : p.allKeys(Profile::KeySelectionRecursive)) { + if (!key.startsWith(fullPrefix)) + continue; + const QString jobPool = key.mid(fullPrefix.size()); + const int limit = p.value(key).toInt(); + if (limit >= 0) + limits.setJobLimit(jobPool, limit); + } + } + return limits; +} + QVariant Preferences::getPreference(const QString &key, const QVariant &defaultValue) const { static const QString keyPrefix = QLatin1String("preferences"); diff --git a/src/lib/corelib/tools/preferences.h b/src/lib/corelib/tools/preferences.h index 07f0edcd7..661b39d7f 100644 --- a/src/lib/corelib/tools/preferences.h +++ b/src/lib/corelib/tools/preferences.h @@ -42,6 +42,7 @@ #include "qbs_export.h" #include "commandechomode.h" +#include "joblimits.h" #include "settings.h" #include @@ -63,6 +64,7 @@ public: CommandEchoMode defaultEchoMode() const; QStringList searchPaths(const QString &baseDir = QString()) const; QStringList pluginPaths(const QString &baseDir = QString()) const; + JobLimits jobLimits() const; private: QVariant getPreference(const QString &key, const QVariant &defaultValue = QVariant()) const; diff --git a/src/lib/corelib/tools/stringconstants.h b/src/lib/corelib/tools/stringconstants.h index f2666e070..6fcf3002b 100644 --- a/src/lib/corelib/tools/stringconstants.h +++ b/src/lib/corelib/tools/stringconstants.h @@ -104,6 +104,8 @@ public: QBS_STRING_CONSTANT(installPrefixProperty, "installPrefix") QBS_STRING_CONSTANT(installDirProperty, "installDir") QBS_STRING_CONSTANT(installSourceBaseProperty, "installSourceBase") + QBS_STRING_CONSTANT(jobCountProperty, "jobCount") + QBS_STRING_CONSTANT(jobPoolProperty, "jobPool") QBS_STRING_CONSTANT(lengthProperty, "length") QBS_STRING_CONSTANT(limitToSubProjectProperty, "limitToSubProject") QBS_STRING_CONSTANT(minimumQbsVersionProperty, "minimumQbsVersion") diff --git a/src/lib/corelib/tools/tools.pri b/src/lib/corelib/tools/tools.pri index bb3a55f12..f9c6be9a5 100644 --- a/src/lib/corelib/tools/tools.pri +++ b/src/lib/corelib/tools/tools.pri @@ -21,6 +21,7 @@ HEADERS += \ $$PWD/generateoptions.h \ $$PWD/id.h \ $$PWD/iosutils.h \ + $$PWD/joblimits.h \ $$PWD/jsliterals.h \ $$PWD/launcherinterface.h \ $$PWD/launcherpackets.h \ @@ -75,6 +76,7 @@ SOURCES += \ $$PWD/filetime.cpp \ $$PWD/generateoptions.cpp \ $$PWD/id.cpp \ + $$PWD/joblimits.cpp \ $$PWD/jsliterals.cpp \ $$PWD/launcherinterface.cpp \ $$PWD/launcherpackets.cpp \ @@ -124,6 +126,7 @@ osx { $$PWD/error.h \ $$PWD/generateoptions.h \ $$PWD/installoptions.h \ + $$PWD/joblimits.h \ $$PWD/preferences.h \ $$PWD/processresult.h \ $$PWD/profile.h \ diff --git a/tests/auto/auto.pro b/tests/auto/auto.pro index f6cc4b26f..1afe48176 100644 --- a/tests/auto/auto.pro +++ b/tests/auto/auto.pro @@ -14,5 +14,6 @@ SUBDIRS += \ blackbox/blackbox-apple.pro \ blackbox/blackbox-clangdb.pro \ blackbox/blackbox-java.pro \ + blackbox/blackbox-joblimits.pro \ blackbox/blackbox-qt.pro \ api diff --git a/tests/auto/auto.qbs b/tests/auto/auto.qbs index 92ab5279f..bf75d0f23 100644 --- a/tests/auto/auto.qbs +++ b/tests/auto/auto.qbs @@ -9,6 +9,7 @@ Project { "blackbox/blackbox-apple.qbs", "blackbox/blackbox-clangdb.qbs", "blackbox/blackbox-java.qbs", + "blackbox/blackbox-joblimits.qbs", "blackbox/blackbox-qt.qbs", "buildgraph/buildgraph.qbs", "cmdlineparser/cmdlineparser.qbs", diff --git a/tests/auto/blackbox/blackbox-joblimits.pro b/tests/auto/blackbox/blackbox-joblimits.pro new file mode 100644 index 000000000..85413473e --- /dev/null +++ b/tests/auto/blackbox/blackbox-joblimits.pro @@ -0,0 +1,18 @@ +TARGET = tst_blackbox-joblimits + +HEADERS = tst_blackboxbase.h +SOURCES = tst_blackboxjoblimits.cpp tst_blackboxbase.cpp +OBJECTS_DIR = joblimits +MOC_DIR = $${OBJECTS_DIR}-moc + +include(../auto.pri) + +DATA_DIRS = testdata-joblimits ../find + +for(data_dir, DATA_DIRS) { + files = $$files($$PWD/$$data_dir/*, true) + win32:files ~= s|\\\\|/|g + for(file, files):!exists($$file/*):FILES += $$file +} + +OTHER_FILES += $$FILES diff --git a/tests/auto/blackbox/blackbox-joblimits.qbs b/tests/auto/blackbox/blackbox-joblimits.qbs new file mode 100644 index 000000000..857e1de7f --- /dev/null +++ b/tests/auto/blackbox/blackbox-joblimits.qbs @@ -0,0 +1,20 @@ +import qbs.Utilities + +QbsAutotest { + testName: "blackbox-joblimits" + Depends { name: "qbs_app" } + Depends { name: "qbs-setup-toolchains" } + Group { + name: "testdata" + prefix: "testdata-joblimits/" + files: ["**/*"] + fileTags: [] + } + files: [ + "../shared.h", + "tst_blackboxbase.cpp", + "tst_blackboxbase.h", + "tst_blackboxjoblimits.cpp", + ] + cpp.defines: base.concat(["SRCDIR=" + Utilities.cStringQuote(path)]) +} diff --git a/tests/auto/blackbox/testdata-joblimits/job-limits/job-limits.qbs b/tests/auto/blackbox/testdata-joblimits/job-limits/job-limits.qbs new file mode 100644 index 000000000..409849681 --- /dev/null +++ b/tests/auto/blackbox/testdata-joblimits/job-limits/job-limits.qbs @@ -0,0 +1,93 @@ +import qbs.TextFile + +Project { + property int projectJobCount + property int productJobCount + property int moduleJobCount + JobLimit { + condition: projectJobCount !== -1 + jobPool: "singleton" + jobCount: projectJobCount + } + JobLimit { + condition: projectJobCount !== -1 + jobPool: "singleton" + jobCount: 100 + } + CppApplication { + name: "tool" + consoleApplication: true + cpp.cxxLanguageVersion: "c++14" + files: "main.cpp" + Group { + fileTagsFilter: "application" + fileTags: "tool_tag" + } + Export { + Rule { + alwaysRun: true + inputs: "tool_in" + explicitlyDependsOnFromDependencies: "tool_tag" + Artifact { filePath: input.completeBaseName + ".out"; fileTags: "tool_out" } + prepare: { + var cmd = new Command(explicitlyDependsOn.tool_tag[0].filePath, + [output.filePath]); + cmd.workingDirectory = product.buildDirectory; + cmd.description = "Running tool"; + cmd.jobPool = "singleton"; + return cmd; + } + } + JobLimit { + condition: project.moduleJobCount !== -1 + jobPool: "singleton" + jobCount: project.moduleJobCount + } + JobLimit { + condition: project.moduleJobCount !== -1 + jobPool: "singleton" + jobCount: 200 + } + } + } + Product { + name: "p" + type: "tool_out" + Depends { name: "tool" } + Rule { + multiplex: true + outputFileTags: "tool_in" + outputArtifacts: { + var artifacts = []; + for (var i = 0; i < 7; ++i) + artifacts.push({filePath: "file" + i + ".in", fileTags: "tool_in"}); + return artifacts; + } + prepare: { + var commands = []; + for (var i = 0; i < outputs.tool_in.length; ++i) { + var cmd = new JavaScriptCommand(); + var output = outputs.tool_in[i]; + cmd.output = output.filePath; + cmd.description = "generating " + output.fileName; + cmd.sourceCode = function() { + var f = new TextFile(output, TextFile.WriteOnly); + f.close(); + } + commands.push(cmd); + }; + return commands; + } + } + JobLimit { + condition: project.productJobCount !== -1 + jobPool: "singleton" + jobCount: project.productJobCount + } + JobLimit { + condition: project.productJobCount !== -1 + jobPool: "singleton" + jobCount: 300 + } + } +} diff --git a/tests/auto/blackbox/testdata-joblimits/job-limits/main.cpp b/tests/auto/blackbox/testdata-joblimits/job-limits/main.cpp new file mode 100644 index 000000000..42796186c --- /dev/null +++ b/tests/auto/blackbox/testdata-joblimits/job-limits/main.cpp @@ -0,0 +1,65 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) +{ + if (argc != 2) { + std::cerr << "tool needs exactly one argument" << std::endl; + return 1; + } + + const std::string lockFilePath = std::string(argv[0]) + ".lock"; + if (std::fopen(lockFilePath.c_str(), "r")) { + std::cerr << "tool is exclusive" << std::endl; + return 2; + } + std::FILE * const lockFile = std::fopen(lockFilePath.c_str(), "w"); + if (!lockFile) { + std::cerr << "cannot create lock file: " << strerror(errno) << std::endl; + return 3; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + fclose(lockFile); + if (std::remove(lockFilePath.c_str()) != 0) { + std::cerr << "cannot remove lock file: " << strerror(errno) << std::endl; + return 4; + } + std::FILE * const output = std::fopen(argv[1], "w"); + if (!output) { + std::cerr << "cannot create output file: " << strerror(errno) << std::endl; + return 5; + } + fclose(output); +} diff --git a/tests/auto/blackbox/tst_blackboxjoblimits.cpp b/tests/auto/blackbox/tst_blackboxjoblimits.cpp new file mode 100644 index 000000000..59e4ffd52 --- /dev/null +++ b/tests/auto/blackbox/tst_blackboxjoblimits.cpp @@ -0,0 +1,174 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "tst_blackboxbase.h" + +#include "../shared.h" +#include + +class TestBlackboxJobLimits : public TestBlackboxBase +{ + Q_OBJECT + +public: + TestBlackboxJobLimits(); + +private slots: + void jobLimits_data(); + void jobLimits(); +}; + +TestBlackboxJobLimits::TestBlackboxJobLimits() + : TestBlackboxBase (SRCDIR "/testdata-joblimits", "blackbox-joblimits") +{ +} + +void TestBlackboxJobLimits::jobLimits_data() +{ + QTest::addColumn("projectJobCount"); + QTest::addColumn("productJobCount"); + QTest::addColumn("moduleJobCount"); + QTest::addColumn("prefsJobCount"); + QTest::addColumn("cliJobCount"); + QTest::addColumn("projectPrecedence"); + QTest::addColumn("expectSuccess"); + for (int projectJobCount = -1; projectJobCount <= 1; ++projectJobCount) { + for (int productJobCount = -1; productJobCount <= 1; ++productJobCount) { + for (int moduleJobCount = -1; moduleJobCount <= 1; ++moduleJobCount) { + for (int prefsJobCount = -1; prefsJobCount <= 1; ++prefsJobCount) { + for (int cliJobCount = -1; cliJobCount <= 1; ++cliJobCount) { + QString description = QString("project:%1/" + "product:%2/module:%3/prefs:%4/cli:%5/project precedence") + .arg(projectJobCount).arg(productJobCount).arg(moduleJobCount) + .arg(prefsJobCount).arg(cliJobCount).toLocal8Bit(); + bool expectSuccess; + switch (productJobCount) { + case 1: expectSuccess = true; break; + case 0: expectSuccess = false; break; + case -1: + switch (projectJobCount) { + case 1: expectSuccess = true; break; + case 0: expectSuccess = false; break; + case -1: + switch (moduleJobCount) { + case 1: expectSuccess = true; break; + case 0: expectSuccess = false; break; + case -1: + switch (cliJobCount) { + case 1: expectSuccess = true; break; + case 0: expectSuccess = false; break; + case -1: expectSuccess = prefsJobCount == 1; break; + } + break; + } + break; + } + break; + } + QTest::newRow(qPrintable(description)) + << projectJobCount << productJobCount << moduleJobCount + << prefsJobCount << cliJobCount << true << expectSuccess; + description = QString("project:%1/" + "product:%2/module:%3/prefs:%4/cli:%5/default precedence") + .arg(projectJobCount).arg(productJobCount).arg(moduleJobCount) + .arg(prefsJobCount).arg(cliJobCount).toLocal8Bit(); + switch (cliJobCount) { + case 1: expectSuccess = true; break; + case 0: expectSuccess = false; break; + case -1: + switch (prefsJobCount) { + case 1: expectSuccess = true; break; + case 0: expectSuccess = false; break; + case -1: + switch (productJobCount) { + case 1: expectSuccess = true; break; + case 0: expectSuccess = false; break; + case -1: + switch (projectJobCount) { + case 1: expectSuccess = true; break; + case 0: expectSuccess = false; break; + case -1: expectSuccess = moduleJobCount == 1; break; + } + break; + } + break; + } + break; + } + QTest::newRow(qPrintable(description)) + << projectJobCount << productJobCount << moduleJobCount + << prefsJobCount << cliJobCount << false << expectSuccess; + } + } + } + } + } +} + +void TestBlackboxJobLimits::jobLimits() +{ + QDir::setCurrent(testDataDir + "/job-limits"); + QFETCH(int, projectJobCount); + QFETCH(int, productJobCount); + QFETCH(int, moduleJobCount); + QFETCH(int, prefsJobCount); + QFETCH(int, cliJobCount); + QFETCH(bool, projectPrecedence); + QFETCH(bool, expectSuccess); + SettingsPtr theSettings = settings(); + qbs::Internal::TemporaryProfile profile("jobLimitsProfile", theSettings.get()); + profile.p.setValue("preferences.jobLimit.singleton", prefsJobCount); + theSettings->sync(); + QbsRunParameters resolveParams("resolve"); + resolveParams.profile = profile.p.name(); + resolveParams.arguments << ("project.projectJobCount:" + QString::number(projectJobCount)) + << ("project.productJobCount:" + QString::number(productJobCount)) + << ("project.moduleJobCount:" + QString::number(moduleJobCount)); + QCOMPARE(runQbs(resolveParams), 0); + QbsRunParameters buildParams; + buildParams.expectFailure = !expectSuccess; + if (cliJobCount != -1) + buildParams.arguments << "--job-limits" << ("singleton:" + QString::number(cliJobCount)); + if (projectPrecedence) + buildParams.arguments << "--enforce-project-job-limits"; + buildParams.profile = profile.p.name(); + QFile::remove(relativeExecutableFilePath("tool") + ".lock"); + const int exitCode = runQbs(buildParams); + if (expectSuccess) + QCOMPARE(exitCode, 0); + else if (exitCode == 0) + QSKIP("no failure with no limit in place, result inconclusive"); + else + QVERIFY2(m_qbsStderr.contains("exclusive"), m_qbsStderr.constData()); + if (exitCode == 0) + QCOMPARE(m_qbsStdout.count("Running tool"), 7); +} + +QTEST_MAIN(TestBlackboxJobLimits) + +#include -- cgit v1.2.3