Add support for job pools

Commands can now be assigned to an arbitrary job pool and a limit for
the number of concurrently running jobs in such pools can be provided in
a number of ways:
- via the build command line: qbs --job-limits linker:1
- via the settings: qbs config preferences.jobLimit.linker 1
- in a project file: JobLimit {
                         jobPool: "linker";
                         jobCount: 1
                     }
We provide two job pools ourselves with the cpp module: "compiler" and
"linker".

[ChangeLog] Added the concept of job pools for limiting concurrent
execution of commands by type

Task-number: QBS-743
Change-Id: Ib3f361dbc73093e342bf0eba0daf2079a2b3a8ce
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@qt.io>
Reviewed-by: Joerg Bornemann <joerg.bornemann@qt.io>

2 files changed, 158 insertions, 0 deletions

diff --git a/tests/auto/blackbox/testdata-joblimits/job-limits/job-limits.qbs b/tests/auto/blackbox/testdata-joblimits/job-limits/job-limits.qbs
new file mode 100644
index 000000000..409849681
--- /dev/null
+++ b/tests/auto/blackbox/testdata-joblimits/job-limits/job-limits.qbs
@@ -0,0 +1,93 @@
+import qbs.TextFile
+
+Project {
+    property int projectJobCount
+    property int productJobCount
+    property int moduleJobCount
+    JobLimit {
+        condition: projectJobCount !== -1
+        jobPool: "singleton"
+        jobCount: projectJobCount
+    }
+    JobLimit {
+        condition: projectJobCount !== -1
+        jobPool: "singleton"
+        jobCount: 100
+    }
+    CppApplication {
+        name: "tool"
+        consoleApplication: true
+        cpp.cxxLanguageVersion: "c++14"
+        files: "main.cpp"
+        Group {
+            fileTagsFilter: "application"
+            fileTags: "tool_tag"
+        }
+        Export {
+            Rule {
+                alwaysRun: true
+                inputs: "tool_in"
+                explicitlyDependsOnFromDependencies: "tool_tag"
+                Artifact { filePath: input.completeBaseName + ".out"; fileTags: "tool_out" }
+                prepare: {
+                    var cmd = new Command(explicitlyDependsOn.tool_tag[0].filePath,
+                                          [output.filePath]);
+                    cmd.workingDirectory = product.buildDirectory;
+                    cmd.description = "Running tool";
+                    cmd.jobPool = "singleton";
+                    return cmd;
+                }
+            }
+            JobLimit {
+                condition: project.moduleJobCount !== -1
+                jobPool: "singleton"
+                jobCount: project.moduleJobCount
+            }
+            JobLimit {
+                condition: project.moduleJobCount !== -1
+                jobPool: "singleton"
+                jobCount: 200
+            }
+        }
+    }
+    Product {
+        name: "p"
+        type: "tool_out"
+        Depends { name: "tool" }
+        Rule {
+            multiplex: true
+            outputFileTags: "tool_in"
+            outputArtifacts: {
+                var artifacts = [];
+                for (var i = 0; i < 7; ++i)
+                    artifacts.push({filePath: "file" + i + ".in", fileTags: "tool_in"});
+                return artifacts;
+            }
+            prepare: {
+                var commands = [];
+                for (var i = 0; i < outputs.tool_in.length; ++i) {
+                    var cmd = new JavaScriptCommand();
+                    var output = outputs.tool_in[i];
+                    cmd.output = output.filePath;
+                    cmd.description = "generating " + output.fileName;
+                    cmd.sourceCode = function() {
+                        var f = new TextFile(output, TextFile.WriteOnly);
+                        f.close();
+                    }
+                    commands.push(cmd);
+                };
+                return commands;
+            }
+        }
+        JobLimit {
+            condition: project.productJobCount !== -1
+            jobPool: "singleton"
+            jobCount: project.productJobCount
+        }
+        JobLimit {
+            condition: project.productJobCount !== -1
+            jobPool: "singleton"
+            jobCount: 300
+        }
+    }
+}
diff --git a/tests/auto/blackbox/testdata-joblimits/job-limits/main.cpp b/tests/auto/blackbox/testdata-joblimits/job-limits/main.cpp
new file mode 100644
index 000000000..42796186c
--- /dev/null
+++ b/tests/auto/blackbox/testdata-joblimits/job-limits/main.cpp
@@ -0,0 +1,65 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qbs.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <chrono>
+#include <cstdio>
+#include <cstring>
+#include <iostream>
+#include <string>
+#include <thread>
+
+int main(int argc, char *argv[])
+{
+    if (argc != 2) {
+        std::cerr << "tool needs exactly one argument" << std::endl;
+        return 1;
+    }
+
+    const std::string lockFilePath = std::string(argv[0]) + ".lock";
+    if (std::fopen(lockFilePath.c_str(), "r")) {
+        std::cerr << "tool is exclusive" << std::endl;
+        return 2;
+    }
+    std::FILE * const lockFile = std::fopen(lockFilePath.c_str(), "w");
+    if (!lockFile) {
+        std::cerr << "cannot create lock file: " << strerror(errno) << std::endl;
+        return 3;
+    }
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    fclose(lockFile);
+    if (std::remove(lockFilePath.c_str()) != 0) {
+        std::cerr << "cannot remove lock file: " << strerror(errno) << std::endl;
+        return 4;
+    }
+    std::FILE * const output = std::fopen(argv[1], "w");
+    if (!output) {
+        std::cerr << "cannot create output file: " << strerror(errno) << std::endl;
+        return 5;
+    }
+    fclose(output);
+}