diff options
| -rw-r--r-- | qbs.qbs | 3 | ||||
| -rw-r--r-- | tests/fuzzy-test/commandlineparser.cpp | 77 | ||||
| -rw-r--r-- | tests/fuzzy-test/commandlineparser.h | 69 | ||||
| -rw-r--r-- | tests/fuzzy-test/fuzzy-test.pro | 10 | ||||
| -rw-r--r-- | tests/fuzzy-test/fuzzy-test.qbs | 16 | ||||
| -rw-r--r-- | tests/fuzzy-test/fuzzytester.cpp | 155 | ||||
| -rw-r--r-- | tests/fuzzy-test/fuzzytester.h | 69 | ||||
| -rw-r--r-- | tests/fuzzy-test/main.cpp | 80 | ||||
| -rw-r--r-- | tests/tests.pro | 2 |
9 files changed, 479 insertions, 2 deletions
@@ -20,7 +20,8 @@ Project { "doc/doc.qbs", "share/share.qbs", "src/src.qbs", - "tests/auto/auto.qbs" + "tests/auto/auto.qbs", + "tests/fuzzy-test/fuzzy-test.qbs", ] SubProject { diff --git a/tests/fuzzy-test/commandlineparser.cpp b/tests/fuzzy-test/commandlineparser.cpp new file mode 100644 index 000000000..e279f8b40 --- /dev/null +++ b/tests/fuzzy-test/commandlineparser.cpp @@ -0,0 +1,77 @@ +/**************************************************************************** +** +** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/legal +** +** This file is part of the Qt Build Suite. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +****************************************************************************/ +#include "commandlineparser.h" + +#include <QFileInfo> + +static QString profileOption() { return "--profile"; } +static QString startCommitOption() { return "--start-commit"; } + +CommandLineParser::CommandLineParser() +{ +} + +void CommandLineParser::parse(const QStringList &commandLine) +{ + m_profile.clear(); + m_startCommit.clear(); + m_commandLine = commandLine; + Q_ASSERT(!m_commandLine.isEmpty()); + m_command = m_commandLine.takeFirst(); + while (!m_commandLine.isEmpty()) { + const QString arg = m_commandLine.takeFirst(); + if (arg == profileOption()) + assignOptionArgument(arg, m_profile); + else if (arg == startCommitOption()) + assignOptionArgument(arg, m_startCommit); + else + throw ParseException(QString::fromLocal8Bit("Unknown parameter '%1'").arg(arg)); + } + if (m_profile.isEmpty()) + throw ParseException("No profile given."); + if (m_startCommit.isEmpty()) + throw ParseException("No start commit given."); +} + +QString CommandLineParser::usageString() const +{ + return QString::fromLocal8Bit("%1 %2 <profile> %3 <start commit>") + .arg(QFileInfo(m_command).fileName(), profileOption(), startCommitOption()); +} + +void CommandLineParser::assignOptionArgument(const QString &option, QString &argument) +{ + if (m_commandLine.isEmpty()) + throw ParseException(QString::fromLocal8Bit("Option '%1' needs an argument.").arg(option)); + argument = m_commandLine.takeFirst(); + if (argument.isEmpty()) { + throw ParseException(QString::fromLocal8Bit("Argument for option '%1' must not be empty.") + .arg(option)); + } +} diff --git a/tests/fuzzy-test/commandlineparser.h b/tests/fuzzy-test/commandlineparser.h new file mode 100644 index 000000000..2d08df7ac --- /dev/null +++ b/tests/fuzzy-test/commandlineparser.h @@ -0,0 +1,69 @@ +/**************************************************************************** +** +** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/legal +** +** This file is part of the Qt Build Suite. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +****************************************************************************/ +#ifndef QBS_FUZZYTEST_COMMANDLINEPARSER_H +#define QBS_FUZZYTEST_COMMANDLINEPARSER_H + +#include <QStringList> + +#include <exception> + +class ParseException : public std::exception +{ +public: + ParseException(const QString &error) : errorMessage(error) { } + ~ParseException() throw() {} + + QString errorMessage; + +private: + const char *what() const throw() { return qPrintable(errorMessage); } +}; + +class CommandLineParser +{ +public: + CommandLineParser(); + + void parse(const QStringList &commandLine); + + QString profile() const { return m_profile; } + QString startCommit() const { return m_startCommit; } + + QString usageString() const; + +private: + void assignOptionArgument(const QString &option, QString &argument); + + QStringList m_commandLine; + QString m_command; + QString m_profile; + QString m_startCommit; +}; + +#endif // Include guard. diff --git a/tests/fuzzy-test/fuzzy-test.pro b/tests/fuzzy-test/fuzzy-test.pro new file mode 100644 index 000000000..41518dcdd --- /dev/null +++ b/tests/fuzzy-test/fuzzy-test.pro @@ -0,0 +1,10 @@ +TARGET = qbs-fuzzy-test +DESTDIR = ../../bin +CONFIG += console +SOURCES = main.cpp \ + commandlineparser.cpp \ + fuzzytester.cpp + +HEADERS += \ + commandlineparser.h \ + fuzzytester.h diff --git a/tests/fuzzy-test/fuzzy-test.qbs b/tests/fuzzy-test/fuzzy-test.qbs new file mode 100644 index 000000000..1951450ae --- /dev/null +++ b/tests/fuzzy-test/fuzzy-test.qbs @@ -0,0 +1,16 @@ +import qbs + +CppApplication { + name: "qbs-fuzzy-test" + destinationDirectory: "bin" + Depends { name: "Qt.core" } + condition: Qt.core.versionMajor >= 5 // We use QDir::removeRecursively() + consoleApplication: true + files: [ + "commandlineparser.cpp", + "commandlineparser.h", + "fuzzytester.cpp", + "fuzzytester.h", + "main.cpp", + ] +} diff --git a/tests/fuzzy-test/fuzzytester.cpp b/tests/fuzzy-test/fuzzytester.cpp new file mode 100644 index 000000000..bee4c2b4f --- /dev/null +++ b/tests/fuzzy-test/fuzzytester.cpp @@ -0,0 +1,155 @@ +/**************************************************************************** +** +** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/legal +** +** This file is part of the Qt Build Suite. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +****************************************************************************/ +#include "fuzzytester.h" + +#include <QDir> +#include <QProcess> + +#include <algorithm> +#include <cstdlib> +#include <ctime> + +FuzzyTester::FuzzyTester() +{ +} + +void FuzzyTester::runTest(const QString &profile, const QString &startCommit) +{ + m_profile = profile; + + runGit(QStringList() << "describe" << "HEAD", &m_headCommit); + + qDebug("Trying to find a buildable commit to start with..."); + const QString workingStartCommit = findWorkingStartCommit(startCommit); + qDebug("Found buildable start commit %s.", qPrintable(workingStartCommit)); + QStringList allCommits = findAllCommits(workingStartCommit); + qDebug("The test set comprises all %d commits between the start commit and HEAD.", + allCommits.count()); + + // Shuffle the initial sequence. Otherwise all invocations of the tool with the same start + // commit would try the same sequence of commits. + std::srand(std::time(0)); + std::random_shuffle(allCommits.begin(), allCommits.end()); + + quint64 run = 0; + QStringList buildSequence(workingStartCommit); + + // This is in effect an infinite loop for all but the tiniest commit sets. + // That's not a problem -- if you want the application to finish early, just kill it. + while (std::next_permutation(allCommits.begin(), allCommits.end())) { + qDebug("Testing permutation %llu...", ++run); + foreach (const QString ¤tCommit, allCommits) { + buildSequence << currentCommit; + checkoutCommit(currentCommit); + QString qbsError; + if (!runQbs(defaultBuildDir(), &qbsError)) { + // An error could be due to the current commit being faulty. Check that it is + // buildable on its own before reporting a qbs error. + const QString otherDir = "fuzzytest-verification-build"; + removeDir(otherDir); + if (runQbs(otherDir)) { + const QString buildSequenceString = buildSequence.join(QLatin1String(",")); + throw TestError(QString::fromLocal8Bit("Found qbs bug with incremental build!\n" + "The error message was: '%1'\n" + "The sequence of commits was: %2.").arg(qbsError, buildSequenceString)); + } + } + } + } +} + +void FuzzyTester::checkoutCommit(const QString &commit) +{ + runGit(QStringList() << "checkout" << commit); +} + +QStringList FuzzyTester::findAllCommits(const QString &startCommit) +{ + QString allCommitsString; + runGit(QStringList() << "log" << (startCommit + "~1.." + m_headCommit) << "--format=format:%h", + &allCommitsString); + return allCommitsString.simplified().split(QLatin1Char(' ')); +} + +QString FuzzyTester::findWorkingStartCommit(const QString &startCommit) +{ + const QStringList allCommits = findAllCommits(startCommit); + QString qbsError; + for (int i = allCommits.count() - 1; i >= 0; --i) { + QString currentCommit = allCommits.at(i); + checkoutCommit(currentCommit); + removeDir(defaultBuildDir()); + if (runQbs(defaultBuildDir(), &qbsError)) + return currentCommit; + } + throw TestError(QString::fromLocal8Bit("Cannot run test: Failed to find a single commit that " + "builds successfully with qbs. The last qbs error was: '%1'").arg(qbsError)); +} + +void FuzzyTester::runGit(const QStringList &arguments, QString *output) +{ + QProcess git; + git.start("git", arguments); + if (!git.waitForStarted()) + throw TestError("Failed to start git. It is expected to be in the PATH."); + if (!git.waitForFinished()) + throw TestError(QString::fromLocal8Bit("git failed: %1").arg(git.errorString())); + if (output) + *output = QString::fromLocal8Bit(git.readAllStandardOutput()).trimmed(); +} + +bool FuzzyTester::runQbs(const QString &buildDir, QString *errorOutput) +{ + if (errorOutput) + errorOutput->clear(); + QProcess qbs; + qbs.start("qbs", QStringList() << "-qq" << "-d" << buildDir << ("profile:" + m_profile)); + if (!qbs.waitForStarted()) + throw TestError("Failed to start qbs. It is expected to be in the PATH."); + if (!qbs.waitForFinished(-1) || qbs.exitCode() != 0) { + if (errorOutput) + *errorOutput = QString::fromLocal8Bit(qbs.readAllStandardError()); + return false; + } + return true; +} + +void FuzzyTester::removeDir(const QString &dirPath) +{ + QDir dir(dirPath); + if (!dir.removeRecursively()) { + throw TestError(QString::fromLocal8Bit("Failed to remove temporary dir '%1'.") + .arg(dir.absolutePath())); + } +} + +QString FuzzyTester::defaultBuildDir() +{ + return "fuzzytest-build"; +} diff --git a/tests/fuzzy-test/fuzzytester.h b/tests/fuzzy-test/fuzzytester.h new file mode 100644 index 000000000..84da8177a --- /dev/null +++ b/tests/fuzzy-test/fuzzytester.h @@ -0,0 +1,69 @@ +/**************************************************************************** +** +** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/legal +** +** This file is part of the Qt Build Suite. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +****************************************************************************/ +#ifndef QBS_FUZZYTESTER_H +#define QBS_FUZZYTESTER_H + +#include <QString> + +#include <exception> + +class TestError { +public: + TestError(const QString &errorMessage) : errorMessage(errorMessage) {} + ~TestError() throw() {} + + QString errorMessage; + +private: + const char *what() const throw() { return qPrintable(errorMessage); } +}; + + +class FuzzyTester +{ +public: + FuzzyTester(); + + void runTest(const QString &profile, const QString &startCommit); + +private: + void checkoutCommit(const QString &commit); + QStringList findAllCommits(const QString &startCommit); + QString findWorkingStartCommit(const QString &startCommit); + void runGit(const QStringList &arguments, QString *output = 0); + bool runQbs(const QString &buildDir, QString *errorOutput = 0); + void removeDir(const QString &dir); + + static QString defaultBuildDir(); + + QString m_profile; + QString m_headCommit; +}; + +#endif // Include guard. diff --git a/tests/fuzzy-test/main.cpp b/tests/fuzzy-test/main.cpp new file mode 100644 index 000000000..d6953123d --- /dev/null +++ b/tests/fuzzy-test/main.cpp @@ -0,0 +1,80 @@ +/**************************************************************************** +** +** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/legal +** +** This file is part of the Qt Build Suite. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +****************************************************************************/ + +#include "commandlineparser.h" +#include "fuzzytester.h" + +#include <QCoreApplication> + +#include <cstdlib> +#include <iostream> + +static bool parseCommandLine(const QStringList &commandLine, QString &profile, + QString &startCommit); +static bool runTest(const QString &profile, const QString &startCommit); + +int main(int argc, char *argv[]) +{ + QCoreApplication app(argc, argv); + + QString profile; + QString startCommit; + if (!parseCommandLine(app.arguments(), profile, startCommit)) + return EXIT_FAILURE; + if (!runTest(profile, startCommit)) + return EXIT_FAILURE; + std::cout << "Test finished successfully." << std::endl; + return EXIT_SUCCESS; +} + +bool parseCommandLine(const QStringList &commandLine, QString &profile, QString &startCommit) +{ + CommandLineParser cmdParser; + try { + cmdParser.parse(commandLine); + } catch (const ParseException &e) { + std::cerr << "Invalid command line: " << qPrintable(e.errorMessage) << std::endl; + std::cerr << "Usage:" << std::endl << qPrintable(cmdParser.usageString()) << std::endl; + return false; + } + profile = cmdParser.profile(); + startCommit = cmdParser.startCommit(); + return true; +} + +bool runTest(const QString &profile, const QString &startCommit) +{ + try { + FuzzyTester().runTest(profile, startCommit); + } catch (const TestError &e) { + std::cerr << qPrintable(e.errorMessage) << std::endl; + return false; + } + return true; +} diff --git a/tests/tests.pro b/tests/tests.pro index cef82e9c7..e85059d70 100644 --- a/tests/tests.pro +++ b/tests/tests.pro @@ -1,3 +1,3 @@ TEMPLATE = subdirs SUBDIRS = auto - +greaterThan(QT_MAJOR_VERSION, 4):SUBDIRS += fuzzy-test # We use QDir::removeRecursively() |