fuzzy tester: Cache information about build results.

We remember the information whether a clean build succeeded. This
enables us to skip potentially expensive verification builds as well as
finding a usable start commit.
Note: This can lead to false positives or negatives if qbs or the build
environment have changed in a way that leads to a formerly succeeding
build failing or vice versa; however, this should be rare and easily
spottable.

Change-Id: I690f7a01fc1174fa50214775b0faa3b550168963
Reviewed-by: Joerg Bornemann <joerg.bornemann@theqtcompany.com>

2 files changed, 60 insertions, 12 deletions

diff --git a/tests/fuzzy-test/fuzzytester.cpp b/tests/fuzzy-test/fuzzytester.cpp
index d465a3bdc..f78ee3f0d 100644
--- a/tests/fuzzy-test/fuzzytester.cpp
+++ b/tests/fuzzy-test/fuzzytester.cpp
@@ -33,6 +33,7 @@
 #include <QElapsedTimer>
 #include <QFile>
 #include <QProcess>
+#include <QSettings>
 
 #include <algorithm>
 #include <cstdlib>
@@ -44,6 +45,12 @@ static QString buildFromScratchActivity() { return "build-from-scratch"; }
 
 FuzzyTester::FuzzyTester()
 {
+    loadSettings();
+}
+
+FuzzyTester::~FuzzyTester()
+{
+    storeSettings();
 }
 
 void FuzzyTester::runTest(const QString &profile, const QString &startCommit,
@@ -153,13 +160,19 @@ QString FuzzyTester::findWorkingStartCommit(const QString &startCommit)
     QString qbsError;
     m_currentActivity = buildFromScratchActivity();
     for (int i = allCommits.count() - 1; i >= 0; --i) {
-        QString currentCommit = allCommits.at(i);
-        checkoutCommit(currentCommit);
+        m_currentCommit = allCommits.at(i);
+        if (m_unbuildableCommits.contains(m_currentCommit)) {
+            qDebug("Skipping known bad commit %s.", qPrintable(m_currentCommit));
+            continue;
+        }
+        checkoutCommit(m_currentCommit);
         removeDir(defaultBuildDir());
-        m_currentCommit = currentCommit;
-        if (runQbs(defaultBuildDir(), QLatin1String("build"), &qbsError))
-            return currentCommit;
-        qDebug("Commit %s is not buildable.", qPrintable(currentCommit));
+        if (runQbs(defaultBuildDir(), QLatin1String("build"), &qbsError)) {
+            m_buildableCommits << m_currentCommit;
+            return m_currentCommit;
+        }
+        qDebug("Commit %s is not buildable.", qPrintable(m_currentCommit));
+        m_unbuildableCommits << m_currentCommit;
     }
     throw TestError(QString::fromLocal8Bit("Cannot run test: Failed to find a single commit that "
             "builds successfully with qbs. The last qbs error was: '%1'").arg(qbsError));
@@ -231,9 +244,22 @@ void FuzzyTester::removeDir(const QString &dirPath)
 
 bool FuzzyTester::doCleanBuild(QString *errorMessage)
 {
+    if (m_unbuildableCommits.contains(m_currentCommit)) {
+        qDebug("Commit is known not to be buildable, not running qbs.");
+        return false;
+    }
+    if (m_buildableCommits.contains(m_currentCommit)) {
+        qDebug("Commit is known to be buildable, not running qbs.");
+        return true;
+    }
     const QString cleanBuildDir = "fuzzytest-verification-build";
     removeDir(cleanBuildDir);
-    return runQbs(cleanBuildDir, QLatin1String("build"), errorMessage);
+    if (runQbs(cleanBuildDir, QLatin1String("build"), errorMessage)) {
+        m_buildableCommits << m_currentCommit;
+        return true;
+    }
+    m_unbuildableCommits << m_currentCommit;
+    return false;
 }
 
 void FuzzyTester::throwIncrementalBuildError(const QString &message,
@@ -254,3 +280,23 @@ QString FuzzyTester::defaultBuildDir()
 {
     return "fuzzytest-build";
 }
+
+
+static QString organization() { return "QtProject"; }
+static QString app() { return "qbs-fuzzy-tester"; }
+static QString unbuildableCommitsKey() { return "unbuildable-commits"; }
+static QString buildableCommitsKey() { return "buildable-commits"; }
+
+void FuzzyTester::loadSettings()
+{
+    QSettings s(organization(), app());
+    m_unbuildableCommits = s.value(unbuildableCommitsKey()).toStringList();
+    m_buildableCommits = s.value(buildableCommitsKey()).toStringList();
+}
+
+void FuzzyTester::storeSettings() const
+{
+    QSettings s(organization(), app());
+    s.setValue(unbuildableCommitsKey(), m_unbuildableCommits);
+    s.setValue(buildableCommitsKey(), m_buildableCommits);
+}
diff --git a/tests/fuzzy-test/fuzzytester.h b/tests/fuzzy-test/fuzzytester.h
index 1ae8a602c..aeaa21e72 100644
--- a/tests/fuzzy-test/fuzzytester.h
+++ b/tests/fuzzy-test/fuzzytester.h
@@ -31,14 +31,10 @@
 #define QBS_FUZZYTESTER_H
 
 #include <QQueue>
-#include <QString>
+#include <QStringList>
 
 #include <exception>
 
-QT_BEGIN_NAMESPACE
-class QStringList;
-QT_END_NAMESPACE
-
 class TestError {
 public:
     TestError(const QString &errorMessage) : errorMessage(errorMessage) {}
@@ -55,6 +51,7 @@ class FuzzyTester
 {
 public:
     FuzzyTester();
+    ~FuzzyTester();
 
     void runTest(const QString &profile, const QString &startCommit, int maxDurationInMinutes,
                  int jobCount, bool log);
@@ -69,6 +66,9 @@ private:
     bool doCleanBuild(QString *errorMessage = 0);
     void throwIncrementalBuildError(const QString &message, const QStringList &commitSequence);
 
+    void loadSettings();
+    void storeSettings() const;
+
     static QString logFilePath(const QString &commit, const QString &activity);
     static QString defaultBuildDir();
 
@@ -79,6 +79,8 @@ private:
     QString m_currentCommit;
     QString m_currentActivity;
     QQueue<QString> m_commitsWithLogFiles;
+    QStringList m_unbuildableCommits;
+    QStringList m_buildableCommits;
 };
 
 #endif // Include guard.