AutoTest: Support some ctest settings

Add some useful settings for ctest based testing. Adapt output
parser accordingly to get at least relevant output and display
significant information inside the visual display.
Current implementation loses or mixes some information when
running jobs in parallel.
Complete output is still available inside the text display.

Fixes: QTCREATORBUG-26029
Change-Id: Id040f83900a6cfd89f5c2aff987278a297aa14c4
Reviewed-by: David Schulz <david.schulz@qt.io>
Reviewed-by: Christian Stenger <christian.stenger@qt.io>

10 files changed, 293 insertions, 11 deletions

diff --git a/src/plugins/autotest/CMakeLists.txt b/src/plugins/autotest/CMakeLists.txt
index 64c2392d38..881b5d2bc8 100644
--- a/src/plugins/autotest/CMakeLists.txt
+++ b/src/plugins/autotest/CMakeLists.txt
@@ -25,6 +25,7 @@ add_qtc_plugin(AutoTest
     catch/catchtestsettings.cpp catch/catchtestsettings.h
     ctest/ctestconfiguration.cpp ctest/ctestconfiguration.h
     ctest/ctestoutputreader.cpp ctest/ctestoutputreader.h
+    ctest/ctestsettings.cpp ctest/ctestsettings.h
     ctest/ctesttool.cpp ctest/ctesttool.h
     ctest/ctesttreeitem.cpp ctest/ctesttreeitem.h
     gtest/gtest_utils.cpp gtest/gtest_utils.h
diff --git a/src/plugins/autotest/autotest.pro b/src/plugins/autotest/autotest.pro
index d7f708d63c..2e0d2ff905 100644
--- a/src/plugins/autotest/autotest.pro
+++ b/src/plugins/autotest/autotest.pro
@@ -9,6 +9,7 @@ SOURCES += \
     autotestplugin.cpp \
     ctest/ctestconfiguration.cpp \
     ctest/ctestoutputreader.cpp \
+    ctest/ctestsettings.cpp \
     ctest/ctesttool.cpp \
     ctest/ctesttreeitem.cpp \
     itestframework.cpp \
@@ -80,6 +81,7 @@ HEADERS += \
     autotestplugin.h \
     ctest/ctestconfiguration.h \
     ctest/ctestoutputreader.h \
+    ctest/ctestsettings.h \
     ctest/ctesttool.h \
     ctest/ctesttreeitem.h \
     itemdatacache.h \
diff --git a/src/plugins/autotest/ctest/ctestoutputreader.cpp b/src/plugins/autotest/ctest/ctestoutputreader.cpp
index 31b41b4134..93315c0bea 100644
--- a/src/plugins/autotest/ctest/ctestoutputreader.cpp
+++ b/src/plugins/autotest/ctest/ctestoutputreader.cpp
@@ -79,9 +79,12 @@ CTestOutputReader::CTestOutputReader(const QFutureInterface<TestResultPtr> &futu
 
 void CTestOutputReader::processOutputLine(const QByteArray &outputLine)
 {
+    static const QRegularExpression verbose("^\\d+: .*$");
     static const QRegularExpression testProject("^Test project (.*)$");
-    static const QRegularExpression testCase("^(test \\d+)|(    Start\\s+\\d+: .*)$");
-    static const QRegularExpression testResult("^\\s*\\d+/\\d+ Test\\s+#\\d+: (.*) (\\.+)\\s*"
+    static const QRegularExpression testCase1("^test (?<current>\\d+)$");
+    static const QRegularExpression testCase2("^    Start\\s+(?<current>\\d+): (?<name>.*)$");
+    static const QRegularExpression testResult("^\\s*(?<first>\\d+/\\d+)? "
+                                               "Test\\s+#(?<current>\\d+): (.*) (\\.+)\\s*"
                                                "(Passed|\\*\\*\\*Failed|\\*\\*\\*Not Run|"
                                                ".*\\*\\*\\*Exception:.*)\\s+(.*) sec$");
     static const QRegularExpression testCrash("^\\s*\\d+/\\d+ Test\\s+#\\d+: (.*) (\\.+)\\s*"
@@ -100,7 +103,9 @@ void CTestOutputReader::processOutputLine(const QByteArray &outputLine)
         operator bool() const { return hasMatch(); }
     };
 
-    if (ExactMatch match = testProject.match(line)) {
+    if (ExactMatch match = verbose.match(line)) { // ignore verbose output for visual display
+        return;
+    } else if (ExactMatch match = testProject.match(line)) {
         if (!m_testName.isEmpty()) // possible?
             sendCompleteInformation();
         m_project = match.captured(1);
@@ -108,13 +113,27 @@ void CTestOutputReader::processOutputLine(const QByteArray &outputLine)
         testResult->setResult(ResultType::TestStart);
         testResult->setDescription(tr("Running tests for %1").arg(m_project));
         reportResult(testResult);
-    } else if (ExactMatch match = testCase.match(line)) {
-        if (!m_testName.isEmpty())
+    } else if (ExactMatch match = testCase1.match(line)) {
+        int current = match.captured("current").toInt();
+        if (m_result != ResultType::Invalid && m_currentTestNo != -1 && current != m_currentTestNo)
+            sendCompleteInformation();
+        m_currentTestNo = -1;
+    } else if (ExactMatch match = testCase2.match(line)) {
+        int current = match.captured("current").toInt();
+        if (m_result != ResultType::Invalid && m_currentTestNo != -1 && current != m_currentTestNo)
             sendCompleteInformation();
+        m_currentTestNo = -1;
     } else if (ExactMatch match = testResult.match(line)) {
-        m_description = match.captured();
-        m_testName = match.captured(1);
-        const QString resultType = match.captured(3);
+        int current = match.captured("current").toInt();
+        if (m_result != ResultType::Invalid && m_currentTestNo != -1 && current != m_currentTestNo)
+            sendCompleteInformation();
+        if (!m_description.isEmpty() && match.captured("first").isEmpty())
+            m_description.append('\n').append(match.captured());
+        else
+            m_description = match.captured();
+        m_currentTestNo = current;
+        m_testName = match.captured(3);
+        const QString resultType = match.captured(5);
         if (resultType == "Passed")
             m_result = ResultType::Pass;
         else if (resultType == "***Failed" || resultType == "***Not Run")
@@ -162,12 +181,19 @@ TestResultPtr CTestOutputReader::createDefaultResult() const
 
 void CTestOutputReader::sendCompleteInformation()
 {
+    // some verbose output we did not ignore
+    if (m_result == ResultType::Invalid) {
+        QTC_CHECK(m_currentTestNo == -1 && m_testName.isEmpty());
+        return;
+    }
+
     TestResultPtr testResult = createDefaultResult();
     testResult->setResult(m_result);
     testResult->setDescription(m_description);
     reportResult(testResult);
     m_testName.clear();
     m_description.clear();
+    m_currentTestNo = -1;
     m_result = ResultType::Invalid;
 }
 
diff --git a/src/plugins/autotest/ctest/ctestoutputreader.h b/src/plugins/autotest/ctest/ctestoutputreader.h
index baff9a9ca1..bbec3f70c7 100644
--- a/src/plugins/autotest/ctest/ctestoutputreader.h
+++ b/src/plugins/autotest/ctest/ctestoutputreader.h
@@ -41,7 +41,9 @@ public:
 protected:
     void processOutputLine(const QByteArray &outputLineWithNewLine) final;
     TestResultPtr createDefaultResult() const final;
+private:
     void sendCompleteInformation();
+    int m_currentTestNo = -1;
     QString m_project;
     QString m_testName;
     QString m_description;
diff --git a/src/plugins/autotest/ctest/ctestsettings.cpp b/src/plugins/autotest/ctest/ctestsettings.cpp
new file mode 100644
index 0000000000..4c8bfb643d
--- /dev/null
+++ b/src/plugins/autotest/ctest/ctestsettings.cpp
@@ -0,0 +1,173 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "ctestsettings.h"
+
+#include "../autotestconstants.h"
+
+#include <utils/layoutbuilder.h>
+
+namespace Autotest {
+namespace Internal {
+
+CTestSettings::CTestSettings()
+{
+    setSettingsGroups("Autotest", "CTest");
+    setAutoApply(false);
+
+    registerAspect(&outputOnFail);
+    outputOnFail.setSettingsKey("OutputOnFail");
+    outputOnFail.setLabelText(tr("Output on failure"));
+    outputOnFail.setDefaultValue(true);
+
+    registerAspect(&outputMode);
+    outputMode.setSettingsKey("OutputMode");
+    outputMode.setLabelText(tr("Output mode"));
+    outputMode.setDisplayStyle(Utils::SelectionAspect::DisplayStyle::ComboBox);
+    outputMode.addOption({tr("Default"), {}, 0});
+    outputMode.addOption({tr("Verbose"), {}, 1});
+    outputMode.addOption({tr("Very Verbose"), {}, 2});
+
+    registerAspect(&repetitionMode);
+    repetitionMode.setSettingsKey("RepetitionMode");
+    repetitionMode.setLabelText(tr("Repetition mode"));
+    repetitionMode.setDisplayStyle(Utils::SelectionAspect::DisplayStyle::ComboBox);
+    repetitionMode.addOption({tr("Until Fail"), {}, 0});
+    repetitionMode.addOption({tr("Until Pass"), {}, 1});
+    repetitionMode.addOption({tr("After Timeout"), {}, 2});
+
+    registerAspect(&repetitionCount);
+    repetitionCount.setSettingsKey("RepetitionCount");
+    repetitionCount.setDefaultValue(1);
+    repetitionCount.setLabelText(tr("Count"));
+    repetitionCount.setToolTip(tr("Number of re-runs for the test."));
+    repetitionCount.setRange(1, 10000);
+
+    registerAspect(&repeat);
+    repeat.setSettingsKey("Repeat");
+
+    registerAspect(&scheduleRandom);
+    scheduleRandom.setSettingsKey("ScheduleRandom");
+    scheduleRandom.setLabelText(tr("Schedule random"));
+
+    registerAspect(&stopOnFailure);
+    stopOnFailure.setSettingsKey("StopOnFail");
+    stopOnFailure.setLabelText(tr("Stop on failure"));
+
+    registerAspect(&parallel);
+    parallel.setSettingsKey("Parallel");
+    parallel.setToolTip(tr("Run tests in parallel mode using given number of jobs."));
+
+    registerAspect(&jobs);
+    jobs.setSettingsKey("Jobs");
+    jobs.setLabelText(tr("Jobs"));
+    jobs.setDefaultValue(1);
+    jobs.setRange(1, 128);
+
+    registerAspect(&testLoad);
+    testLoad.setSettingsKey("TestLoad");
+    testLoad.setLabelText(tr("Test load"));
+    testLoad.setToolTip(tr("Try not to start tests when they may cause CPU load to pass a "
+                           "threshold."));
+
+    registerAspect(&threshold);
+    threshold.setSettingsKey("Threshold");
+    threshold.setLabelText(tr("Threshold"));
+    threshold.setDefaultValue(1);
+    threshold.setRange(1, 128);
+    threshold.setEnabler(&testLoad);
+}
+
+QStringList CTestSettings::activeSettingsAsOptions() const
+{
+    QStringList options;
+    if (outputOnFail.value())
+        options << "--output-on-failure";
+    switch (outputMode.value()) {
+    case 1: options << "-V"; break;
+    case 2: options << "-VV"; break;
+    default: break;
+    }
+    if (repeat.value()) {
+        QString repeatOption;
+        switch (repetitionMode.value()) {
+        case 0: repeatOption = "until-fail"; break;
+        case 1: repeatOption = "until-pass"; break;
+        case 2: repeatOption = "after-timeout"; break;
+        default: break;
+        }
+        if (!repeatOption.isEmpty()) {
+            repeatOption.append(':');
+            repeatOption.append(QString::number(repetitionCount.value()));
+            options << "--repeat" << repeatOption;
+        }
+    }
+    if (scheduleRandom.value())
+        options << "--schedule-random";
+    if (stopOnFailure.value())
+        options << "--stop-on-failure";
+    if (parallel.value()) {
+        options << "-j" << QString::number(jobs.value());
+        if (testLoad.value())
+            options << "--test-load" << QString::number(threshold.value());
+    }
+    return options;
+}
+
+CTestSettingsPage::CTestSettingsPage(CTestSettings *settings, Utils::Id settingsId)
+{
+    setId(settingsId);
+    setCategory(Constants::AUTOTEST_SETTINGS_CATEGORY);
+    setDisplayName(tr("CTest"));
+
+    setSettings(settings);
+
+    setLayouter([settings](QWidget *widget) {
+        CTestSettings &s = *settings;
+        using namespace Utils::Layouting;
+        const Break nl;
+
+        Form form {
+            Row {s.outputOnFail}, nl,
+            Row {s.scheduleRandom}, nl,
+            Row {s.stopOnFailure}, nl,
+            Row {s.outputMode}, nl,
+            Group {
+                Title(tr("Repeat tests"), &s.repeat), nl,
+                Row {s.repetitionMode, s.repetitionCount},
+            }, nl,
+            Group {
+                Title(tr("Run in parallel"), &s.parallel), nl,
+                Row {s.jobs}, nl,
+                Row {s.testLoad, s.threshold}
+            }
+        };
+
+        Column {Row {Column { form , Stretch() }, Stretch() } }.attachTo(widget);
+    });
+}
+
+} // namespace Internal
+} // namespace Autotest
diff --git a/src/plugins/autotest/ctest/ctestsettings.h b/src/plugins/autotest/ctest/ctestsettings.h
new file mode 100644
index 0000000000..5683987e9d
--- /dev/null
+++ b/src/plugins/autotest/ctest/ctestsettings.h
@@ -0,0 +1,66 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <coreplugin/dialogs/ioptionspage.h>
+
+#include <utils/aspects.h>
+
+namespace Autotest {
+namespace Internal {
+
+class CTestSettings : public Utils::AspectContainer
+{
+    Q_DECLARE_TR_FUNCTIONS(Autotest::Internal::CTestSettings)
+public:
+    CTestSettings();
+
+    QStringList activeSettingsAsOptions() const;
+
+    Utils::IntegerAspect repetitionCount;
+    Utils::SelectionAspect repetitionMode;
+    Utils::SelectionAspect outputMode;
+    Utils::BoolAspect outputOnFail;
+    Utils::BoolAspect stopOnFailure;
+    Utils::BoolAspect scheduleRandom;
+    Utils::BoolAspect repeat;
+    // FIXME.. this makes the outputreader fail to get all results correctly for visual display
+    Utils::BoolAspect parallel;
+    Utils::IntegerAspect jobs;
+    Utils::BoolAspect testLoad;
+    Utils::IntegerAspect threshold;
+};
+
+class CTestSettingsPage final : public Core::IOptionsPage
+{
+    Q_DECLARE_TR_FUNCTIONS(Autotest::Internal::CTestSettingsPage)
+public:
+    CTestSettingsPage(CTestSettings *settings, Utils::Id settingsId);
+};
+
+} // namespace Internal
+} // namespace Autotest
+
diff --git a/src/plugins/autotest/ctest/ctesttool.h b/src/plugins/autotest/ctest/ctesttool.h
index 30cce7df53..ff0d3b7522 100644
--- a/src/plugins/autotest/ctest/ctesttool.h
+++ b/src/plugins/autotest/ctest/ctesttool.h
@@ -26,6 +26,7 @@
 #pragma once
 
 #include "../itestframework.h"
+#include "ctestsettings.h"
 
 namespace Autotest {
 namespace Internal {
@@ -42,6 +43,12 @@ public:
 protected:
     const char *name() const final;
     ITestTreeItem *createRootNode() final;
+
+private:
+    ITestSettings *testSettings() override { return &m_settings; }
+
+    CTestSettings m_settings;
+    CTestSettingsPage m_settingsPage{&m_settings, settingsId()};
 };
 
 } // namespace Internal
diff --git a/src/plugins/autotest/ctest/ctesttreeitem.cpp b/src/plugins/autotest/ctest/ctesttreeitem.cpp
index d9230461fb..5e08d62c48 100644
--- a/src/plugins/autotest/ctest/ctesttreeitem.cpp
+++ b/src/plugins/autotest/ctest/ctesttreeitem.cpp
@@ -26,6 +26,7 @@
 #include "ctesttreeitem.h"
 
 #include "ctestconfiguration.h"
+#include "ctestsettings.h"
 
 #include "../autotestplugin.h"
 #include "../itestframework.h"
@@ -107,8 +108,8 @@ QList<ITestConfiguration *> CTestTreeItem::testConfigurationsFor(const QStringLi
 
     const ProjectExplorer::BuildSystem *buildSystem = target->buildSystem();
     QStringList options{"--timeout", QString::number(AutotestPlugin::settings()->timeout / 1000)};
-    // TODO add ctest options from settings?
-    options << "--output-on-failure";
+    auto ctestSettings = static_cast<CTestSettings *>(testBase()->testSettings());
+    options << ctestSettings->activeSettingsAsOptions();
     Utils::CommandLine command = buildSystem->commandLineForTests(selected, options);
     if (command.executable().isEmpty())
         return {};
diff --git a/src/plugins/autotest/testframeworkmanager.cpp b/src/plugins/autotest/testframeworkmanager.cpp
index b7ec3dc022..0352a3dd61 100644
--- a/src/plugins/autotest/testframeworkmanager.cpp
+++ b/src/plugins/autotest/testframeworkmanager.cpp
@@ -125,6 +125,10 @@ void TestFrameworkManager::synchronizeSettings(QSettings *s)
         if (ITestSettings *fSettings = framework->testSettings())
             fSettings->readSettings(s);
     }
+    for (ITestTool *testTool : qAsConst(m_registeredTestTools)) {
+        if (ITestSettings *tSettings = testTool->testSettings())
+            tSettings->readSettings(s);
+    }
 }
 
 } // namespace Autotest
diff --git a/src/plugins/autotest/testresultmodel.cpp b/src/plugins/autotest/testresultmodel.cpp
index c4060f4d42..ad8b4bd177 100644
--- a/src/plugins/autotest/testresultmodel.cpp
+++ b/src/plugins/autotest/testresultmodel.cpp
@@ -129,7 +129,7 @@ static bool isSignificant(ResultType type)
     case ResultType::MessageCurrentTest:
     case ResultType::Application:
     case ResultType::Invalid:
-        QTC_ASSERT_STRING("Got unexpedted type in isSignificant check");
+        QTC_ASSERT_STRING("Got unexpected type in isSignificant check");
         return false;
     default:
         return true;