Add command and AutotestRunner timeout

Task-number: QBS-1454
Change-Id: I6e2514d10cca0cba0a14456ecd2abfb495539ee4
Reviewed-by: Christian Kandeler <christian.kandeler@qt.io>
Reviewed-by: Denis Shienkov <denis.shienkov@gmail.com>

9 files changed, 226 insertions, 0 deletions

diff --git a/tests/auto/api/testdata/timeout-js/timeout.qbs b/tests/auto/api/testdata/timeout-js/timeout.qbs
new file mode 100644
index 000000000..26aa4ce87
--- /dev/null
+++ b/tests/auto/api/testdata/timeout-js/timeout.qbs
@@ -0,0 +1,20 @@
+Product {
+    type: "product-under-test"
+    Rule {
+        multiplex: true
+        Artifact {
+            filePath: "output.txt"
+            fileTags: "product-under-test"
+        }
+        prepare: {
+            var cmd = new JavaScriptCommand();
+            cmd.description = "Running infinite loop";
+            cmd.sourceCode = function() {
+                while (true)
+                    ;
+            }
+            cmd.timeout = 3;
+            return cmd;
+        }
+    }
+}
diff --git a/tests/auto/api/testdata/timeout-process/main.cpp b/tests/auto/api/testdata/timeout-process/main.cpp
new file mode 100644
index 000000000..f9b9336ba
--- /dev/null
+++ b/tests/auto/api/testdata/timeout-process/main.cpp
@@ -0,0 +1,37 @@
+/****************************************************************************
+**
+** Copyright (C) 2019 Jochen Ulrich <jochenulrich@t-online.de>
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qbs.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <chrono>
+#include <thread>
+
+int main()
+{
+    std::this_thread::sleep_for(std::chrono::seconds(700));
+    return 0;
+}
+
diff --git a/tests/auto/api/testdata/timeout-process/timeout.qbs b/tests/auto/api/testdata/timeout-process/timeout.qbs
new file mode 100644
index 000000000..9f1491ff4
--- /dev/null
+++ b/tests/auto/api/testdata/timeout-process/timeout.qbs
@@ -0,0 +1,24 @@
+Project {
+    CppApplication {
+        type: "application"
+        consoleApplication: true // suppress bundle generation
+        files: "main.cpp"
+        name: "infinite-loop"
+    }
+
+    Product {
+        type: "product-under-test"
+        name: "caller"
+        Depends { name: "infinite-loop" }
+        Rule {
+            inputsFromDependencies: "application"
+            outputFileTags: "product-under-test"
+            prepare: {
+                var cmd = new Command(inputs["application"][0].filePath);
+                cmd.description = "Calling application that runs forever";
+                cmd.timeout = 3;
+                return cmd;
+            }
+        }
+    }
+}
diff --git a/tests/auto/api/tst_api.cpp b/tests/auto/api/tst_api.cpp
index aac0f3e7e..b47d3b5c8 100644
--- a/tests/auto/api/tst_api.cpp
+++ b/tests/auto/api/tst_api.cpp
@@ -1,6 +1,7 @@
 /****************************************************************************
 **
 ** Copyright (C) 2016 The Qt Company Ltd.
+** Copyright (C) 2019 Jochen Ulrich <jochenulrich@t-online.de>
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of Qbs.
@@ -2829,6 +2830,69 @@ void TestApi::targetArtifactStatus()
     QCOMPARE(product.targetArtifacts().size(), enableTagging ? 2 : 1);
 }
 
+void TestApi::timeout()
+{
+    QFETCH(QString, projectDirName);
+    QFETCH(qint64, expectedMaxRunTime);
+    const auto setupParams = defaultSetupParameters(projectDirName + "/timeout.qbs");
+    std::unique_ptr<qbs::SetupProjectJob> setupJob{
+            qbs::Project().setupProject(setupParams, m_logSink, nullptr)};
+    waitForFinished(setupJob.get());
+    QVERIFY2(!setupJob->error().hasError(), qPrintable(setupJob->error().toString()));
+    auto project = setupJob->project();
+    const auto products = project.projectData().products();
+    QList<qbs::ProductData> helperProducts;
+    qbs::ProductData productUnderTest;
+    for (const auto &product : products) {
+        if (!product.type().contains(QLatin1String("product-under-test")))
+            helperProducts.append(product);
+        else
+            productUnderTest = product;
+    }
+    const std::unique_ptr<qbs::BuildJob> buildHelpersJob{
+            project.buildSomeProducts(helperProducts, qbs::BuildOptions())};
+    QVERIFY(waitForFinished(buildHelpersJob.get(), testTimeoutInMsecs()));
+    if (buildHelpersJob->error().hasError()) {
+        qDebug().noquote() << buildHelpersJob->error().toString();
+        QFAIL("Could not build helper products");
+    }
+
+    QElapsedTimer timer;
+    timer.start();
+    const std::unique_ptr<qbs::BuildJob> buildJob(project.buildOneProduct(productUnderTest,
+                                                                          qbs::BuildOptions()));
+    const auto testAbortTimeout = 20 * 1000ll;
+    /* We add an additional buffer to the expectedMaxRunTime because Qbs can take some
+     * time (>1 second) to finish after the command has been cancelled.
+     */
+    expectedMaxRunTime += 4 * 1000ll;
+    Q_ASSERT_X(testAbortTimeout > expectedMaxRunTime,
+               Q_FUNC_INFO,
+               "testAbortTimeout must be larger than the expectedMaxRunTime. Else the "
+               "test might be cancelled to early although the code is working correctly.");
+    QTimer::singleShot(testAbortTimeout, buildJob.get(), &qbs::AbstractJob::cancel);
+    QVERIFY(waitForFinished(buildJob.get(), testTimeoutInMsecs()));
+    const auto actualRunTime = timer.elapsed();
+    QVERIFY(buildJob->error().hasError());
+    const auto errorString = buildJob->error().toString();
+    QVERIFY(errorString.contains("cancel"));
+    QVERIFY(errorString.contains("timeout"));
+    if (actualRunTime > expectedMaxRunTime)
+        qDebug() << actualRunTime << "vs." << expectedMaxRunTime;
+    QVERIFY(actualRunTime < expectedMaxRunTime);
+}
+
+void TestApi::timeout_data()
+{
+    QTest::addColumn<QString>("projectDirName");
+    QTest::addColumn<qint64>("expectedMaxRunTime");
+    QTest::newRow("JS Command") << QString("timeout-js") << 3 * 1000ll;
+    /* The timeout is 3 seconds. Qbs will try to terminate the process for 3 seconds and then kill
+     * it.
+     */
+    QTest::newRow("Process Command") << QString("timeout-process") << (3 + 3) * 1000ll;
+}
+
 void TestApi::toolInModule()
 {
     QVariantMap overrides({std::make_pair("qbs.installRoot", m_workingDataDir
diff --git a/tests/auto/api/tst_api.h b/tests/auto/api/tst_api.h
index c04e9feb0..ce1678133 100644
--- a/tests/auto/api/tst_api.h
+++ b/tests/auto/api/tst_api.h
@@ -1,6 +1,7 @@
 /****************************************************************************
 **
 ** Copyright (C) 2016 The Qt Company Ltd.
+** Copyright (C) 2019 Jochen Ulrich <jochenulrich@t-online.de>
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of Qbs.
@@ -142,6 +143,8 @@ private slots:
     void subProjects();
     void targetArtifactStatus_data();
     void targetArtifactStatus();
+    void timeout();
+    void timeout_data();
     void toolInModule();
     void trackAddQObjectHeader();
     void trackRemoveQObjectHeader();
diff --git a/tests/auto/blackbox/testdata/autotest-timeout/autotests-timeout.qbs b/tests/auto/blackbox/testdata/autotest-timeout/autotests-timeout.qbs
new file mode 100644
index 000000000..2cc20c3fd
--- /dev/null
+++ b/tests/auto/blackbox/testdata/autotest-timeout/autotests-timeout.qbs
@@ -0,0 +1,9 @@
+Project {
+    CppApplication {
+        name: "testApp"
+        type: ["application", "autotest"]
+        Depends { name: "autotest" }
+        files: "test-main.cpp"
+    }
+    AutotestRunner {}
+}
diff --git a/tests/auto/blackbox/testdata/autotest-timeout/test-main.cpp b/tests/auto/blackbox/testdata/autotest-timeout/test-main.cpp
new file mode 100644
index 000000000..4bb1ca27c
--- /dev/null
+++ b/tests/auto/blackbox/testdata/autotest-timeout/test-main.cpp
@@ -0,0 +1,37 @@
+/****************************************************************************
+**
+** Copyright (C) 2019 Jochen Ulrich <jochenulrich@t-online.de>
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qbs.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <chrono>
+#include <thread>
+
+int main()
+{
+    std::this_thread::sleep_for(std::chrono::seconds(5));
+    return 0;
+}
+
diff --git a/tests/auto/blackbox/tst_blackbox.cpp b/tests/auto/blackbox/tst_blackbox.cpp
index f87889155..edc2b51c8 100644
--- a/tests/auto/blackbox/tst_blackbox.cpp
+++ b/tests/auto/blackbox/tst_blackbox.cpp
@@ -1,6 +1,7 @@
 /****************************************************************************
 **
 ** Copyright (C) 2016 The Qt Company Ltd.
+** Copyright (C) 2019 Jochen Ulrich <jochenulrich@t-online.de>
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of Qbs.
@@ -5624,6 +5625,34 @@ void TestBlackbox::autotestWithDependencies()
              && m_qbsStdout.contains("i am the helper"), m_qbsStdout.constData());
 }
 
+void TestBlackbox::autotestTimeout()
+{
+    QFETCH(QStringList, resolveParams);
+    QFETCH(bool, expectFailure);
+    QDir::setCurrent(testDataDir + "/autotest-timeout");
+    QbsRunParameters resolveParameters("resolve", resolveParams);
+    QCOMPARE(runQbs(resolveParameters), 0);
+    QbsRunParameters buildParameters(QStringList({"-p", "autotest-runner"}));
+    buildParameters.expectFailure = expectFailure;
+    if (expectFailure) {
+        QVERIFY(runQbs(buildParameters) != 0);
+        QVERIFY(m_qbsStderr.contains("cancelled") && m_qbsStderr.contains("timeout"));
+    }
+    else
+        QVERIFY(runQbs(buildParameters) == 0);
+}
+
+void TestBlackbox::autotestTimeout_data()
+{
+    QTest::addColumn<QStringList>("resolveParams");
+    QTest::addColumn<bool>("expectFailure");
+    QTest::newRow("no timeout") << QStringList() << false;
+    QTest::newRow("timeout on test") << QStringList({"products.testApp.autotest.timeout:2"})
+            << true;
+    QTest::newRow("timeout on runner") << QStringList({"products.autotest-runner.timeout:2"})
+            << true;
+}
+
 void TestBlackbox::autotests_data()
 {
     QTest::addColumn<QString>("evilPropertySpec");
diff --git a/tests/auto/blackbox/tst_blackbox.h b/tests/auto/blackbox/tst_blackbox.h
index 0a14c418c..4c1268912 100644
--- a/tests/auto/blackbox/tst_blackbox.h
+++ b/tests/auto/blackbox/tst_blackbox.h
@@ -1,6 +1,7 @@
 /****************************************************************************
 **
 ** Copyright (C) 2016 The Qt Company Ltd.
+** Copyright (C) 2019 Jochen Ulrich <jochenulrich@t-online.de>
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of Qbs.
@@ -48,6 +49,8 @@ private slots:
     void artifactScanning();
     void assembly();
     void autotestWithDependencies();
+    void autotestTimeout();
+    void autotestTimeout_data();
     void autotests_data();
     void autotests();
     void auxiliaryInputsFromDependencies();