Make fuzzy test more thorough.

"Successful" incremental builds can also be an error, if a clean build
fails for the same project.

Change-Id: I9f52d840fbda53a906c039e1ee0db2c656db6f8e
Reviewed-by: Joerg Bornemann <joerg.bornemann@digia.com>

2 files changed, 34 insertions, 10 deletions

diff --git a/tests/fuzzy-test/fuzzytester.cpp b/tests/fuzzy-test/fuzzytester.cpp
index e8624c16f..5f9888520 100644
--- a/tests/fuzzy-test/fuzzytester.cpp
+++ b/tests/fuzzy-test/fuzzytester.cpp
@@ -77,17 +77,19 @@ void FuzzyTester::runTest(const QString &profile, const QString &startCommit)
             bool success = runQbs(defaultBuildDir(), QLatin1String("resolve"), &qbsError);
             if (success)
                 success = runQbs(defaultBuildDir(), QLatin1String("build"), &qbsError);
-            if (!success) {
-                // An error could be due to the current commit being faulty. Check that it is
-                // buildable on its own before reporting a qbs error.
+            if (success) {
+                if (!doCleanBuild()) {
+                    const QString message = QString::fromLocal8Bit("An incremental build succeeded "
+                            "with a commit for which a clean build failed.");
+                    throwIncrementalBuildError(message, buildSequence);
+                }
+            } else {
                 qDebug("Incremental build failed. Checking whether clean build works...");
-                const QString otherDir = "fuzzytest-verification-build";
-                removeDir(otherDir);
-                if (runQbs(otherDir, QLatin1String("build"))) {
-                    const QString buildSequenceString = buildSequence.join(QLatin1String(","));
-                    throw TestError(QString::fromLocal8Bit("Found qbs bug with incremental build!\n"
-                            "The error message was: '%1'\n"
-                            "The sequence of commits was: %2.").arg(qbsError, buildSequenceString));
+                if (doCleanBuild()) {
+                    const QString message = QString::fromLocal8Bit("An incremental build failed "
+                            "with a commit for which a clean build succeeded.\n"
+                            "The qbs error message was: '%1'").arg(qbsError);
+                    throwIncrementalBuildError(message, buildSequence);
                 } else {
                     qDebug("Clean build also fails. Continuing.");
                 }
@@ -167,6 +169,22 @@ void FuzzyTester::removeDir(const QString &dirPath)
     }
 }
 
+bool FuzzyTester::doCleanBuild()
+{
+    const QString cleanBuildDir = "fuzzytest-verification-build";
+    removeDir(cleanBuildDir);
+    return runQbs(cleanBuildDir, QLatin1String("build"));
+}
+
+void FuzzyTester::throwIncrementalBuildError(const QString &message,
+                                             const QStringList &commitSequence)
+{
+    const QString commitSequenceString = commitSequence.join(QLatin1String(","));
+    throw TestError(QString::fromLocal8Bit("Found qbs bug with incremental build!\n"
+            "%1\n"
+            "The sequence of commits was: %2.").arg(message, commitSequenceString));
+}
+
 QString FuzzyTester::defaultBuildDir()
 {
     return "fuzzytest-build";
diff --git a/tests/fuzzy-test/fuzzytester.h b/tests/fuzzy-test/fuzzytester.h
index 2f002cd0a..9934aa679 100644
--- a/tests/fuzzy-test/fuzzytester.h
+++ b/tests/fuzzy-test/fuzzytester.h
@@ -33,6 +33,10 @@
 
 #include <exception>
 
+QT_BEGIN_NAMESPACE
+class QStringList;
+QT_END_NAMESPACE
+
 class TestError {
 public:
     TestError(const QString &errorMessage) : errorMessage(errorMessage) {}
@@ -59,6 +63,8 @@ private:
     void runGit(const QStringList &arguments, QString *output = 0);
     bool runQbs(const QString &buildDir, const QString &command, QString *errorOutput = 0);
     void removeDir(const QString &dir);
+    bool doCleanBuild();
+    void throwIncrementalBuildError(const QString &message, const QStringList &commitSequence);
 
     static QString defaultBuildDir();