From 3b36dcc4c68cafc514e0e3a37144a8c039763589 Mon Sep 17 00:00:00 2001 From: Christian Stenger Date: Fri, 17 Apr 2020 13:45:38 +0200 Subject: AutoTest: Provide catch settings Users can now modify some settings of Catch tests. Task-number: QTCREATORBUG-19740 Change-Id: I47e64a43f22fbf783cbf7b256e498d9037533e9a Reviewed-by: David Schulz --- src/plugins/autotest/CMakeLists.txt | 2 + src/plugins/autotest/autotest.pro | 5 + src/plugins/autotest/catch/catchconfiguration.cpp | 29 ++ src/plugins/autotest/catch/catchframework.h | 10 +- src/plugins/autotest/catch/catchtestsettings.cpp | 94 +++++++ src/plugins/autotest/catch/catchtestsettings.h | 62 +++++ .../autotest/catch/catchtestsettingspage.cpp | 113 ++++++++ src/plugins/autotest/catch/catchtestsettingspage.h | 42 +++ .../autotest/catch/catchtestsettingspage.ui | 303 +++++++++++++++++++++ 9 files changed, 659 insertions(+), 1 deletion(-) create mode 100644 src/plugins/autotest/catch/catchtestsettings.cpp create mode 100644 src/plugins/autotest/catch/catchtestsettings.h create mode 100644 src/plugins/autotest/catch/catchtestsettingspage.cpp create mode 100644 src/plugins/autotest/catch/catchtestsettingspage.h create mode 100644 src/plugins/autotest/catch/catchtestsettingspage.ui (limited to 'src/plugins/autotest') diff --git a/src/plugins/autotest/CMakeLists.txt b/src/plugins/autotest/CMakeLists.txt index 1d8e6dde6bd..619501af8ba 100644 --- a/src/plugins/autotest/CMakeLists.txt +++ b/src/plugins/autotest/CMakeLists.txt @@ -26,6 +26,8 @@ add_qtc_plugin(AutoTest catch/catchframework.h catch/catchframework.cpp catch/catchoutputreader.h catch/catchoutputreader.cpp catch/catchresult.h catch/catchresult.cpp catch/catchtestparser.h catch/catchtestparser.cpp catch/catchtreeitem.h catch/catchtreeitem.cpp + catch/catchtestsettings.cpp catch/catchtestsettings.h + catch/catchtestsettingspage.cpp catch/catchtestsettingspage.h catch/catchtestsettingspage.ui gtest/gtest_utils.cpp gtest/gtest_utils.h gtest/gtestconfiguration.cpp gtest/gtestconfiguration.h gtest/gtestconstants.h diff --git a/src/plugins/autotest/autotest.pro b/src/plugins/autotest/autotest.pro index 2633883b09b..7082a385864 100644 --- a/src/plugins/autotest/autotest.pro +++ b/src/plugins/autotest/autotest.pro @@ -33,6 +33,8 @@ SOURCES += \ catch/catchoutputreader.cpp \ catch/catchresult.cpp \ catch/catchtestparser.cpp \ + catch/catchtestsettings.cpp \ + catch/catchtestsettingspage.cpp \ catch/catchtreeitem.cpp \ gtest/gtestconfiguration.cpp \ gtest/gtestparser.cpp \ @@ -104,6 +106,8 @@ HEADERS += \ catch/catchoutputreader.h \ catch/catchresult.h \ catch/catchtestparser.h \ + catch/catchtestsettings.h \ + catch/catchtestsettingspage.h \ catch/catchtreeitem.h \ gtest/gtestconfiguration.h \ gtest/gtestparser.h \ @@ -150,6 +154,7 @@ RESOURCES += \ FORMS += \ testsettingspage.ui \ boost/boosttestsettingspage.ui \ + catch/catchtestsettingspage.ui \ qtest/qttestsettingspage.ui \ gtest/gtestsettingspage.ui diff --git a/src/plugins/autotest/catch/catchconfiguration.cpp b/src/plugins/autotest/catch/catchconfiguration.cpp index ae112b73e93..51b2946a4fe 100644 --- a/src/plugins/autotest/catch/catchconfiguration.cpp +++ b/src/plugins/autotest/catch/catchconfiguration.cpp @@ -24,8 +24,10 @@ #include "catchconfiguration.h" #include "catchoutputreader.h" +#include "catchtestsettings.h" #include "../autotestplugin.h" +#include "../itestframework.h" #include "../testsettings.h" namespace Autotest { @@ -100,6 +102,33 @@ QStringList CatchConfiguration::argumentsForTestRunner(QStringList *omitted) con ' ', QString::SkipEmptyParts), omitted); } + auto settings = dynamic_cast(framework()->frameworkSettings()); + if (!settings) + return arguments; + + if (settings->abortAfterChecked) + arguments << "-x" << QString::number(settings->abortAfter); + if (settings->samplesChecked) + arguments << "--benchmark-samples" << QString::number(settings->benchmarkSamples); + if (settings->resamplesChecked) + arguments << "--benchmark-resamples" << QString::number(settings->benchmarkResamples); + if (settings->warmupChecked) + arguments << "--benchmark-warmup-time" << QString::number(settings->benchmarkWarmupTime); + if (settings->confidenceIntervalChecked) + arguments << "--benchmark-confidence-interval" << QString::number(settings->confidenceInterval); + if (settings->noAnalysis) + arguments << "--benchmark-no-analysis"; + if (settings->showSuccess) + arguments << "-s"; + if (settings->noThrow) + arguments << "-e"; + if (settings->visibleWhitespace) + arguments << "-i"; + if (settings->warnOnEmpty) + arguments << "-w" << "NoAssertions"; + + if (isDebugRunMode() && settings->breakOnFailure) + arguments << "-b"; return arguments; } diff --git a/src/plugins/autotest/catch/catchframework.h b/src/plugins/autotest/catch/catchframework.h index 5f226da4bc6..178f5ebc14e 100644 --- a/src/plugins/autotest/catch/catchframework.h +++ b/src/plugins/autotest/catch/catchframework.h @@ -26,6 +26,9 @@ #include "../itestframework.h" +#include "catchtestsettings.h" +#include "catchtestsettingspage.h" + namespace Autotest { namespace Internal { @@ -40,7 +43,12 @@ public: protected: ITestParser *createTestParser() override; TestTreeItem *createRootNode() override; + +private: + IFrameworkSettings * frameworkSettings() override { return &m_settings; } + CatchTestSettings m_settings; + CatchTestSettingsPage m_settingsPage{&m_settings, settingsId()}; }; -} // namepsace Internal +} // namespace Internal } // namespace Autotest diff --git a/src/plugins/autotest/catch/catchtestsettings.cpp b/src/plugins/autotest/catch/catchtestsettings.cpp new file mode 100644 index 00000000000..276a54985b4 --- /dev/null +++ b/src/plugins/autotest/catch/catchtestsettings.cpp @@ -0,0 +1,94 @@ +/**************************************************************************** +** +** Copyright (C) 2020 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#include "catchtestsettings.h" + +namespace Autotest { +namespace Internal { + +static const char abortAfterKey[] = "AbortAfter"; +static const char abortAfterCheckedKey[] = "AbortChecked"; +static const char benchmarkSamplesKey[] = "BenchSamples"; +static const char samplesCheckedKey[] = "SamplesChecked"; +static const char benchmarkResamplesKey[] = "BenchResamples"; +static const char resamplesCheckedKey[] = "ResamplesChecked"; +static const char benchmarConfidenceIntervalKey[] = "BenchConfInt"; +static const char confIntCheckedKey[] = "ConfIntChecked"; +static const char benchmarWarmupTimeKey[] = "BenchWarmup"; +static const char warmupCheckedKey[] = "WarmupChecked"; +static const char noAnalysisKey[] = "NoAnalysis"; +static const char showSuccessKey[] = "ShowSuccess"; +static const char breakOnFailureKey[] = "BreakOnFailure"; +static const char noThrowKey[] = "NoThrow"; +static const char visibleWhitespaceKey[] = "VisibleWS"; +static const char warnOnEmptyKey[] = "WarnEmpty"; + +QString CatchTestSettings::name() const +{ + return QString("Catch2"); +} + +void CatchTestSettings::toFrameworkSettings(QSettings *s) const +{ + s->setValue(abortAfterCheckedKey, abortAfterChecked); + s->setValue(abortAfterKey, abortAfter); + s->setValue(samplesCheckedKey, samplesChecked); + s->setValue(benchmarkSamplesKey, benchmarkSamples); + s->setValue(resamplesCheckedKey, resamplesChecked); + s->setValue(benchmarkResamplesKey, benchmarkResamples); + s->setValue(confIntCheckedKey, confidenceIntervalChecked); + s->setValue(benchmarConfidenceIntervalKey, confidenceInterval); + s->setValue(warmupCheckedKey, warmupChecked); + s->setValue(benchmarWarmupTimeKey, benchmarkWarmupTime); + s->setValue(noAnalysisKey, noAnalysis); + s->setValue(showSuccessKey, showSuccess); + s->setValue(breakOnFailureKey, breakOnFailure); + s->setValue(noThrowKey, noThrow); + s->setValue(visibleWhitespaceKey, visibleWhitespace); + s->setValue(warnOnEmptyKey, warnOnEmpty); +} + +void CatchTestSettings::fromFrameworkSettings(const QSettings *s) +{ + abortAfterChecked = s->value(abortAfterCheckedKey, false).toBool(); + abortAfter = s->value(abortAfterKey, 0).toInt(); + samplesChecked = s->value(samplesCheckedKey, false).toBool(); + benchmarkSamples = s->value(benchmarkSamplesKey, 100).toInt(); + resamplesChecked = s->value(resamplesCheckedKey, false).toBool(); + benchmarkResamples = s->value(benchmarkResamplesKey, 100000).toInt(); + confidenceIntervalChecked = s->value(confIntCheckedKey, false).toBool(); + confidenceInterval = s->value(benchmarConfidenceIntervalKey, 0.95).toDouble(); + warmupChecked = s->value(warmupCheckedKey, false).toBool(); + benchmarkWarmupTime = s->value(benchmarWarmupTimeKey, 0).toInt(); + noAnalysis = s->value(noAnalysisKey, false).toBool(); + showSuccess = s->value(showSuccessKey, false).toBool(); + breakOnFailure = s->value(breakOnFailureKey, true).toBool(); + noThrow = s->value(noThrowKey, false).toBool(); + visibleWhitespace = s->value(visibleWhitespaceKey, false).toBool(); + warnOnEmpty = s->value(warnOnEmptyKey, false).toBool(); +} + +} // namespace Internal +} // namespace Autotest diff --git a/src/plugins/autotest/catch/catchtestsettings.h b/src/plugins/autotest/catch/catchtestsettings.h new file mode 100644 index 00000000000..71c8d9eea5e --- /dev/null +++ b/src/plugins/autotest/catch/catchtestsettings.h @@ -0,0 +1,62 @@ +/**************************************************************************** +** +** Copyright (C) 2020 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#pragma once + +#include "../iframeworksettings.h" + +namespace Autotest { +namespace Internal { + +class CatchTestSettings : public IFrameworkSettings +{ +public: + CatchTestSettings() = default; + QString name() const override; + + int abortAfter = 0; + int benchmarkSamples = 0; + int benchmarkResamples = 0; + double confidenceInterval = 0; + int benchmarkWarmupTime = 0; + bool abortAfterChecked = false; + bool samplesChecked = false; + bool resamplesChecked = false; + bool confidenceIntervalChecked = false; + bool warmupChecked = false; + bool noAnalysis = false; + bool showSuccess = false; + bool breakOnFailure = true; + bool noThrow = false; + bool visibleWhitespace = false; + bool warnOnEmpty = false; + +protected: + void toFrameworkSettings(QSettings *s) const override; + void fromFrameworkSettings(const QSettings *s) override; +}; + +} // namespace Internal +} // namespace Autotest diff --git a/src/plugins/autotest/catch/catchtestsettingspage.cpp b/src/plugins/autotest/catch/catchtestsettingspage.cpp new file mode 100644 index 00000000000..8c54dbe8c8a --- /dev/null +++ b/src/plugins/autotest/catch/catchtestsettingspage.cpp @@ -0,0 +1,113 @@ +/**************************************************************************** +** +** Copyright (C) 2020 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#include "catchtestsettingspage.h" +#include "catchtestsettings.h" +#include "ui_catchtestsettingspage.h" +#include "../autotestconstants.h" + +#include + +namespace Autotest { +namespace Internal { + +class CatchTestSettingsWidget final : public Core::IOptionsPageWidget +{ + Q_DECLARE_TR_FUNCTIONS(Autotest::Internal::CatchTestSettingsWidget) +public: + explicit CatchTestSettingsWidget(CatchTestSettings *settings); + void apply() override; +private: + Ui::CatchTestSettingsPage m_ui; + CatchTestSettings *m_settings; +}; + +CatchTestSettingsWidget::CatchTestSettingsWidget(CatchTestSettings *settings) + : m_settings(settings) +{ + m_ui.setupUi(this); + + m_ui.abortSB->setEnabled(m_settings->abortAfterChecked); + m_ui.samplesSB->setEnabled(m_settings->samplesChecked), + m_ui.resamplesSB->setEnabled(m_settings->resamplesChecked); + m_ui.confIntSB->setEnabled(m_settings->confidenceIntervalChecked); + m_ui.warmupSB->setEnabled(m_settings->warmupChecked); + + connect(m_ui.abortCB, &QCheckBox::toggled, m_ui.abortSB, &QSpinBox::setEnabled); + connect(m_ui.samplesCB, &QCheckBox::toggled, m_ui.samplesSB, &QSpinBox::setEnabled); + connect(m_ui.resamplesCB, &QCheckBox::toggled, m_ui.resamplesSB, &QSpinBox::setEnabled); + connect(m_ui.confIntCB, &QCheckBox::toggled, m_ui.confIntSB, &QDoubleSpinBox::setEnabled); + connect(m_ui.warmupCB, &QCheckBox::toggled, m_ui.warmupSB, &QSpinBox::setEnabled); + + m_ui.showSuccessCB->setChecked(m_settings->showSuccess); + m_ui.breakOnFailCB->setChecked(m_settings->breakOnFailure); + m_ui.noThrowCB->setChecked(m_settings->noThrow); + m_ui.visibleWhiteCB->setChecked(m_settings->visibleWhitespace); + m_ui.warnOnEmpty->setChecked(m_settings->warnOnEmpty); + m_ui.noAnalysisCB->setChecked(m_settings->noAnalysis); + m_ui.abortCB->setChecked(m_settings->abortAfterChecked); + m_ui.abortSB->setValue(m_settings->abortAfter); + m_ui.samplesCB->setChecked(m_settings->samplesChecked); + m_ui.samplesSB->setValue(m_settings->benchmarkSamples); + m_ui.resamplesCB->setChecked(m_settings->resamplesChecked); + m_ui.resamplesSB->setValue(m_settings->benchmarkResamples); + m_ui.confIntCB->setChecked(m_settings->confidenceIntervalChecked); + m_ui.confIntSB->setValue(m_settings->confidenceInterval); + m_ui.warmupCB->setChecked(m_settings->warmupChecked); + m_ui.warmupSB->setValue(m_settings->benchmarkWarmupTime); +} + +void CatchTestSettingsWidget::apply() +{ + m_settings->showSuccess = m_ui.showSuccessCB->isChecked(); + m_settings->breakOnFailure = m_ui.breakOnFailCB->isChecked(); + m_settings->noThrow = m_ui.noThrowCB->isChecked(); + m_settings->visibleWhitespace = m_ui.visibleWhiteCB->isChecked(); + m_settings->warnOnEmpty = m_ui.warnOnEmpty->isChecked(); + m_settings->noAnalysis = m_ui.noAnalysisCB->isChecked(); + m_settings->abortAfterChecked = m_ui.abortCB->isChecked(); + m_settings->abortAfter = m_ui.abortSB->value(); + m_settings->samplesChecked = m_ui.samplesCB->isChecked(); + m_settings->benchmarkSamples = m_ui.samplesSB->value(); + m_settings->resamplesChecked = m_ui.resamplesCB->isChecked(); + m_settings->benchmarkResamples = m_ui.resamplesSB->value(); + m_settings->confidenceIntervalChecked = m_ui.confIntCB->isChecked(); + m_settings->confidenceInterval = m_ui.confIntSB->value(); + m_settings->warmupChecked = m_ui.warmupCB->isChecked(); + m_settings->benchmarkWarmupTime = m_ui.warmupSB->value(); + + m_settings->toSettings(Core::ICore::settings()); +} + +CatchTestSettingsPage::CatchTestSettingsPage(CatchTestSettings *settings, Core::Id settingsId) +{ + setId(settingsId); + setCategory(Constants::AUTOTEST_SETTINGS_CATEGORY); + setDisplayName(QCoreApplication::translate("CatchTestFramework", "Catch Test")); + setWidgetCreator([settings] { return new CatchTestSettingsWidget(settings); }); +} + +} // namespace Internal +} // namespace Autotest diff --git a/src/plugins/autotest/catch/catchtestsettingspage.h b/src/plugins/autotest/catch/catchtestsettingspage.h new file mode 100644 index 00000000000..1c659c7f10b --- /dev/null +++ b/src/plugins/autotest/catch/catchtestsettingspage.h @@ -0,0 +1,42 @@ +/**************************************************************************** +** +** Copyright (C) 2020 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#pragma once + +#include + +namespace Autotest { +namespace Internal { + +class CatchTestSettings; + +class CatchTestSettingsPage : public Core::IOptionsPage +{ +public: + CatchTestSettingsPage(CatchTestSettings *settings, Core::Id settingsId); +}; + +} // namespace Internal +} // namespace Autotest diff --git a/src/plugins/autotest/catch/catchtestsettingspage.ui b/src/plugins/autotest/catch/catchtestsettingspage.ui new file mode 100644 index 00000000000..429d4482b6c --- /dev/null +++ b/src/plugins/autotest/catch/catchtestsettingspage.ui @@ -0,0 +1,303 @@ + + + CatchTestSettingsPage + + + + 0 + 0 + 314 + 323 + + + + Confidence interval used for bootstrapping. + + + + + + + + + + Show success for tests. + + + Show success + + + + + + + Turns failures into debugger breakpoints. + + + Break on failure while debugging + + + true + + + + + + + Skips all assertions that test for thrown exceptions. + + + Skip throwing assertions + + + + + + + Makes whitespace visible. + + + Visualize whitespace + + + + + + + Warns if a test section does not check any assertion. + + + Warn on empty tests + + + + + + + + + Aborts after the specified number of failures. + + + Abort after + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + 1 + + + 9999 + + + + + + + + + + + Number of samples to collect while running benchmarks. + + + Benchmark samples + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + 999999 + + + 100 + + + + + + + + + + + Benchmark resamples + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + Number of resamples for bootstrapping. + + + 9999999 + + + 100000 + + + + + + + + + + + Benchmark confidence interval + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + 1.000000000000000 + + + 0.050000000000000 + + + 0.950000000000000 + + + + + + + + + + + Warmup time for each test. + + + Benchmark warmup time + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + ms + + + 10000 + + + + + + + + + Disables statistical analysis and bootstrapping. + + + Disable analysis + + + + + + + + + Qt::Horizontal + + + + 20 + 20 + + + + + + + + + + Qt::Vertical + + + + 20 + 20 + + + + + + + + + -- cgit v1.2.3