Introduce tool for fuzzy testing.

Incremental builds are the likeliest source of problems, as well as the
one that is the hardest to reproduce. This new tool turns a given project
into a quasi-autotest by checking out random commits and doing incremental
builds on them. In case of a build error that is not reproducible with
a clean build, it aborts and reports the offending sequence of commits.
Any qbs project that comes in a git repository is supported.

Change-Id: I9f6f9172ce03140341ed25859092a8ed32d4a17d
Reviewed-by: Joerg Bornemann <joerg.bornemann@digia.com>

7 files changed, 476 insertions, 0 deletions

diff --git a/tests/fuzzy-test/commandlineparser.cpp b/tests/fuzzy-test/commandlineparser.cpp
new file mode 100644
index 000000000..e279f8b40
--- /dev/null
+++ b/tests/fuzzy-test/commandlineparser.cpp
@@ -0,0 +1,77 @@
+/****************************************************************************
+**
+** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#include "commandlineparser.h"
+
+#include <QFileInfo>
+
+static QString profileOption() { return "--profile"; }
+static QString startCommitOption() { return "--start-commit"; }
+
+CommandLineParser::CommandLineParser()
+{
+}
+
+void CommandLineParser::parse(const QStringList &commandLine)
+{
+    m_profile.clear();
+    m_startCommit.clear();
+    m_commandLine = commandLine;
+    Q_ASSERT(!m_commandLine.isEmpty());
+    m_command = m_commandLine.takeFirst();
+    while (!m_commandLine.isEmpty()) {
+        const QString arg = m_commandLine.takeFirst();
+        if (arg == profileOption())
+            assignOptionArgument(arg, m_profile);
+        else if (arg == startCommitOption())
+            assignOptionArgument(arg, m_startCommit);
+        else
+            throw ParseException(QString::fromLocal8Bit("Unknown parameter '%1'").arg(arg));
+    }
+    if (m_profile.isEmpty())
+        throw ParseException("No profile given.");
+    if (m_startCommit.isEmpty())
+        throw ParseException("No start commit given.");
+}
+
+QString CommandLineParser::usageString() const
+{
+    return QString::fromLocal8Bit("%1 %2 <profile> %3 <start commit>")
+            .arg(QFileInfo(m_command).fileName(), profileOption(), startCommitOption());
+}
+
+void CommandLineParser::assignOptionArgument(const QString &option, QString &argument)
+{
+    if (m_commandLine.isEmpty())
+        throw ParseException(QString::fromLocal8Bit("Option '%1' needs an argument.").arg(option));
+    argument = m_commandLine.takeFirst();
+    if (argument.isEmpty()) {
+        throw ParseException(QString::fromLocal8Bit("Argument for option '%1' must not be empty.")
+                             .arg(option));
+    }
+}
diff --git a/tests/fuzzy-test/commandlineparser.h b/tests/fuzzy-test/commandlineparser.h
new file mode 100644
index 000000000..2d08df7ac
--- /dev/null
+++ b/tests/fuzzy-test/commandlineparser.h
@@ -0,0 +1,69 @@
+/****************************************************************************
+**
+** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#ifndef QBS_FUZZYTEST_COMMANDLINEPARSER_H
+#define QBS_FUZZYTEST_COMMANDLINEPARSER_H
+
+#include <QStringList>
+
+#include <exception>
+
+class ParseException : public std::exception
+{
+public:
+    ParseException(const QString &error) : errorMessage(error) { }
+    ~ParseException() throw() {}
+
+    QString errorMessage;
+
+private:
+    const char *what() const throw() { return qPrintable(errorMessage); }
+};
+
+class CommandLineParser
+{
+public:
+    CommandLineParser();
+
+    void parse(const QStringList &commandLine);
+
+    QString profile() const { return m_profile; }
+    QString startCommit() const { return m_startCommit; }
+
+    QString usageString() const;
+
+private:
+    void assignOptionArgument(const QString &option, QString &argument);
+
+    QStringList m_commandLine;
+    QString m_command;
+    QString m_profile;
+    QString m_startCommit;
+};
+
+#endif // Include guard.
diff --git a/tests/fuzzy-test/fuzzy-test.pro b/tests/fuzzy-test/fuzzy-test.pro
new file mode 100644
index 000000000..41518dcdd
--- /dev/null
+++ b/tests/fuzzy-test/fuzzy-test.pro
@@ -0,0 +1,10 @@
+TARGET = qbs-fuzzy-test
+DESTDIR = ../../bin
+CONFIG += console
+SOURCES = main.cpp \
+    commandlineparser.cpp \
+    fuzzytester.cpp
+
+HEADERS += \
+    commandlineparser.h \
+    fuzzytester.h
diff --git a/tests/fuzzy-test/fuzzy-test.qbs b/tests/fuzzy-test/fuzzy-test.qbs
new file mode 100644
index 000000000..1951450ae
--- /dev/null
+++ b/tests/fuzzy-test/fuzzy-test.qbs
@@ -0,0 +1,16 @@
+import qbs
+
+CppApplication {
+    name: "qbs-fuzzy-test"
+    destinationDirectory: "bin"
+    Depends { name: "Qt.core" }
+    condition: Qt.core.versionMajor >= 5 // We use QDir::removeRecursively()
+    consoleApplication: true
+    files: [
+        "commandlineparser.cpp",
+        "commandlineparser.h",
+        "fuzzytester.cpp",
+        "fuzzytester.h",
+        "main.cpp",
+    ]
+}
diff --git a/tests/fuzzy-test/fuzzytester.cpp b/tests/fuzzy-test/fuzzytester.cpp
new file mode 100644
index 000000000..bee4c2b4f
--- /dev/null
+++ b/tests/fuzzy-test/fuzzytester.cpp
@@ -0,0 +1,155 @@
+/****************************************************************************
+**
+** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#include "fuzzytester.h"
+
+#include <QDir>
+#include <QProcess>
+
+#include <algorithm>
+#include <cstdlib>
+#include <ctime>
+
+FuzzyTester::FuzzyTester()
+{
+}
+
+void FuzzyTester::runTest(const QString &profile, const QString &startCommit)
+{
+    m_profile = profile;
+
+    runGit(QStringList() << "describe" << "HEAD", &m_headCommit);
+
+    qDebug("Trying to find a buildable commit to start with...");
+    const QString workingStartCommit = findWorkingStartCommit(startCommit);
+    qDebug("Found buildable start commit %s.", qPrintable(workingStartCommit));
+    QStringList allCommits = findAllCommits(workingStartCommit);
+    qDebug("The test set comprises all %d commits between the start commit and HEAD.",
+           allCommits.count());
+
+    // Shuffle the initial sequence. Otherwise all invocations of the tool with the same start
+    // commit would try the same sequence of commits.
+    std::srand(std::time(0));
+    std::random_shuffle(allCommits.begin(), allCommits.end());
+
+    quint64 run = 0;
+    QStringList buildSequence(workingStartCommit);
+
+    // This is in effect an infinite loop for all but the tiniest commit sets.
+    // That's not a problem -- if you want the application to finish early, just kill it.
+    while (std::next_permutation(allCommits.begin(), allCommits.end())) {
+        qDebug("Testing permutation %llu...", ++run);
+        foreach (const QString &currentCommit, allCommits) {
+            buildSequence << currentCommit;
+            checkoutCommit(currentCommit);
+            QString qbsError;
+            if (!runQbs(defaultBuildDir(), &qbsError)) {
+                // An error could be due to the current commit being faulty. Check that it is
+                // buildable on its own before reporting a qbs error.
+                const QString otherDir = "fuzzytest-verification-build";
+                removeDir(otherDir);
+                if (runQbs(otherDir)) {
+                    const QString buildSequenceString = buildSequence.join(QLatin1String(","));
+                    throw TestError(QString::fromLocal8Bit("Found qbs bug with incremental build!\n"
+                            "The error message was: '%1'\n"
+                            "The sequence of commits was: %2.").arg(qbsError, buildSequenceString));
+                }
+            }
+        }
+    }
+}
+
+void FuzzyTester::checkoutCommit(const QString &commit)
+{
+    runGit(QStringList() << "checkout" << commit);
+}
+
+QStringList FuzzyTester::findAllCommits(const QString &startCommit)
+{
+    QString allCommitsString;
+    runGit(QStringList() << "log" << (startCommit + "~1.." + m_headCommit) << "--format=format:%h",
+           &allCommitsString);
+    return allCommitsString.simplified().split(QLatin1Char(' '));
+}
+
+QString FuzzyTester::findWorkingStartCommit(const QString &startCommit)
+{
+    const QStringList allCommits = findAllCommits(startCommit);
+    QString qbsError;
+    for (int i = allCommits.count() - 1; i >= 0; --i) {
+        QString currentCommit = allCommits.at(i);
+        checkoutCommit(currentCommit);
+        removeDir(defaultBuildDir());
+        if (runQbs(defaultBuildDir(), &qbsError))
+            return currentCommit;
+    }
+    throw TestError(QString::fromLocal8Bit("Cannot run test: Failed to find a single commit that "
+            "builds successfully with qbs. The last qbs error was: '%1'").arg(qbsError));
+}
+
+void FuzzyTester::runGit(const QStringList &arguments, QString *output)
+{
+    QProcess git;
+    git.start("git", arguments);
+    if (!git.waitForStarted())
+        throw TestError("Failed to start git. It is expected to be in the PATH.");
+    if (!git.waitForFinished())
+        throw TestError(QString::fromLocal8Bit("git failed: %1").arg(git.errorString()));
+    if (output)
+        *output = QString::fromLocal8Bit(git.readAllStandardOutput()).trimmed();
+}
+
+bool FuzzyTester::runQbs(const QString &buildDir, QString *errorOutput)
+{
+    if (errorOutput)
+        errorOutput->clear();
+    QProcess qbs;
+    qbs.start("qbs", QStringList() << "-qq" << "-d" << buildDir << ("profile:" + m_profile));
+    if (!qbs.waitForStarted())
+        throw TestError("Failed to start qbs. It is expected to be in the PATH.");
+    if (!qbs.waitForFinished(-1) || qbs.exitCode() != 0) {
+        if (errorOutput)
+            *errorOutput = QString::fromLocal8Bit(qbs.readAllStandardError());
+        return false;
+    }
+    return true;
+}
+
+void FuzzyTester::removeDir(const QString &dirPath)
+{
+    QDir dir(dirPath);
+    if (!dir.removeRecursively()) {
+        throw TestError(QString::fromLocal8Bit("Failed to remove temporary dir '%1'.")
+                        .arg(dir.absolutePath()));
+    }
+}
+
+QString FuzzyTester::defaultBuildDir()
+{
+    return "fuzzytest-build";
+}
diff --git a/tests/fuzzy-test/fuzzytester.h b/tests/fuzzy-test/fuzzytester.h
new file mode 100644
index 000000000..84da8177a
--- /dev/null
+++ b/tests/fuzzy-test/fuzzytester.h
@@ -0,0 +1,69 @@
+/****************************************************************************
+**
+** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#ifndef QBS_FUZZYTESTER_H
+#define QBS_FUZZYTESTER_H
+
+#include <QString>
+
+#include <exception>
+
+class TestError {
+public:
+    TestError(const QString &errorMessage) : errorMessage(errorMessage) {}
+    ~TestError() throw() {}
+
+    QString errorMessage;
+
+private:
+    const char *what() const throw() { return qPrintable(errorMessage); }
+};
+
+
+class FuzzyTester
+{
+public:
+    FuzzyTester();
+
+    void runTest(const QString &profile, const QString &startCommit);
+
+private:
+    void checkoutCommit(const QString &commit);
+    QStringList findAllCommits(const QString &startCommit);
+    QString findWorkingStartCommit(const QString &startCommit);
+    void runGit(const QStringList &arguments, QString *output = 0);
+    bool runQbs(const QString &buildDir, QString *errorOutput = 0);
+    void removeDir(const QString &dir);
+
+    static QString defaultBuildDir();
+
+    QString m_profile;
+    QString m_headCommit;
+};
+
+#endif // Include guard.
diff --git a/tests/fuzzy-test/main.cpp b/tests/fuzzy-test/main.cpp
new file mode 100644
index 000000000..d6953123d
--- /dev/null
+++ b/tests/fuzzy-test/main.cpp
@@ -0,0 +1,80 @@
+/****************************************************************************
+**
+** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+
+#include "commandlineparser.h"
+#include "fuzzytester.h"
+
+#include <QCoreApplication>
+
+#include <cstdlib>
+#include <iostream>
+
+static bool parseCommandLine(const QStringList &commandLine, QString &profile,
+                             QString &startCommit);
+static bool runTest(const QString &profile, const QString &startCommit);
+
+int main(int argc, char *argv[])
+{
+    QCoreApplication app(argc, argv);
+
+    QString profile;
+    QString startCommit;
+    if (!parseCommandLine(app.arguments(), profile, startCommit))
+        return EXIT_FAILURE;
+    if (!runTest(profile, startCommit))
+        return EXIT_FAILURE;
+    std::cout << "Test finished successfully." << std::endl;
+    return EXIT_SUCCESS;
+}
+
+bool parseCommandLine(const QStringList &commandLine, QString &profile, QString &startCommit)
+{
+    CommandLineParser cmdParser;
+    try {
+        cmdParser.parse(commandLine);
+    } catch (const ParseException &e) {
+        std::cerr << "Invalid command line: " << qPrintable(e.errorMessage) << std::endl;
+        std::cerr << "Usage:" << std::endl << qPrintable(cmdParser.usageString()) << std::endl;
+        return false;
+    }
+    profile = cmdParser.profile();
+    startCommit = cmdParser.startCommit();
+    return true;
+}
+
+bool runTest(const QString &profile, const QString &startCommit)
+{
+    try {
+        FuzzyTester().runTest(profile, startCommit);
+    } catch (const TestError &e) {
+        std::cerr << qPrintable(e.errorMessage) << std::endl;
+        return false;
+    }
+    return true;
+}