Remove ability to run tests in random order.

Remove the undocumented feature that allows test functions to be
executed in random order.  The feature was designed to expose unintended
dependencies between test functions -- test functions are only supposed
to depend on the initTestCase() and init() functions.

Aside from the lack of documentation, there are a number of problems
with this feature.  Most importantly, running the tests in random order
has only a 50% chance of exposing dependencies between test functions.
A better strategy would be to run the test functions in reverse order
and complain if that produces different results to running the tests in
the normal order.

Additionally, the random order is not deterministic, so even if a
dependency is exposed during a test run, there's no guarantee that it
will be exposed again.  The feature allows the user to optionally
supply a random seed to make the "random" order deterministic, but as
rand() implementations are not identical across platforms, even that
does not guarantee that dependencies between test functions will be
exposed deterministically.

Change-Id: I39eac34c532ccb988116778bbc5ab05d835874c5
Reviewed-on: http://codereview.qt.nokia.com/3720
Reviewed-by: Qt Sanity Bot <qt_sanity_bot@ovi.com>
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>

12 files changed, 11 insertions, 171 deletions

diff --git a/src/testlib/qabstracttestlogger_p.h b/src/testlib/qabstracttestlogger_p.h
index e592249e71..3ac6787754 100644
--- a/src/testlib/qabstracttestlogger_p.h
+++ b/src/testlib/qabstracttestlogger_p.h
@@ -96,8 +96,6 @@ public:
     virtual void addMessage(MessageTypes type, const char *message,
                             const char *file = 0, int line = 0) = 0;
 
-    virtual void registerRandomSeed(unsigned int seed) = 0;
-
     void outputString(const char *msg);
 
 private:
diff --git a/src/testlib/qplaintestlogger.cpp b/src/testlib/qplaintestlogger.cpp
index 44ec792400..bd1bc11582 100644
--- a/src/testlib/qplaintestlogger.cpp
+++ b/src/testlib/qplaintestlogger.cpp
@@ -345,7 +345,6 @@ void QPlainTestLogger::printBenchmarkResult(const QBenchmarkResult &result)
 }
 
 QPlainTestLogger::QPlainTestLogger()
-: randomSeed(9), hasRandomSeed(false)
 {
 #if defined(Q_OS_WIN) && !defined(Q_OS_WINCE)
     InitializeCriticalSection(&QTest::outputCriticalSection);
@@ -368,17 +367,10 @@ void QPlainTestLogger::startLogging(const char *filename)
         QTest::qt_snprintf(buf, sizeof(buf), "Testing %s\n",
                            QTestResult::currentTestObjectName());
     } else {
-        if (hasRandomSeed) {
-            QTest::qt_snprintf(buf, sizeof(buf),
-                             "********* Start testing of %s *********\n"
-                             "Config: Using QTest library " QTEST_VERSION_STR
-                             ", Qt %s, Random seed %d\n", QTestResult::currentTestObjectName(), qVersion(), randomSeed);
-        } else {
-            QTest::qt_snprintf(buf, sizeof(buf),
-                             "********* Start testing of %s *********\n"
-                             "Config: Using QTest library " QTEST_VERSION_STR
-                             ", Qt %s\n", QTestResult::currentTestObjectName(), qVersion());
-        }
+        QTest::qt_snprintf(buf, sizeof(buf),
+                           "********* Start testing of %s *********\n"
+                           "Config: Using QTest library " QTEST_VERSION_STR
+                           ", Qt %s\n", QTestResult::currentTestObjectName(), qVersion());
     }
     outputMessage(buf);
 }
@@ -440,10 +432,4 @@ void QPlainTestLogger::addMessage(MessageTypes type, const char *message,
     printMessage(QTest::messageType2String(type), message, file, line);
 }
 
-void QPlainTestLogger::registerRandomSeed(unsigned int seed)
-{
-    randomSeed = seed;
-    hasRandomSeed = true;
-}
-
 QT_END_NAMESPACE
diff --git a/src/testlib/qplaintestlogger_p.h b/src/testlib/qplaintestlogger_p.h
index 5fa33e6b51..baa1e24828 100644
--- a/src/testlib/qplaintestlogger_p.h
+++ b/src/testlib/qplaintestlogger_p.h
@@ -75,11 +75,8 @@ public:
 
     void addMessage(MessageTypes type, const char *message,
                     const char *file = 0, int line = 0);
-    void registerRandomSeed(unsigned int seed);
-private:
-    unsigned int randomSeed;
-    bool hasRandomSeed;
 
+private:
     void printMessage(const char *type, const char *msg, const char *file = 0, int line = 0);
     void outputMessage(const char *str);
     void printBenchmarkResult(const QBenchmarkResult &result);
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index b6aaa34756..3802794281 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -869,10 +869,7 @@ namespace QTest
     static int keyDelay = -1;
     static int mouseDelay = -1;
     static int eventDelay = -1;
-    static bool randomOrder = false;
     static int keyVerbose = -1;
-    static unsigned int seed = 0;
-    static bool seedSet = false;
 #if defined(Q_OS_UNIX) && !defined(Q_OS_SYMBIAN)
     static bool noCrashHandler = false;
 #endif
@@ -958,37 +955,6 @@ int Q_TESTLIB_EXPORT defaultKeyDelay()
     return keyDelay;
 }
 
-void seedRandom()
-{
-    static bool randomSeeded = false;
-    if (!randomSeeded) {
-        if (!QTest::seedSet) {
-            QElapsedTimer timer;
-            timer.start();
-            QTest::seed = timer.msecsSinceReference();
-        }
-        qsrand(QTest::seed);
-        randomSeeded = true;
-    }
-}
-
-template<typename T>
-void swap(T * array, int pos, int otherPos)
-{
-    T tmp = array[pos];
-    array[pos] = array[otherPos];
-    array[otherPos] = tmp;
-}
-
-template<typename T>
-static void randomizeList(T * array, int size)
-{
-    for (int i = 0; i != size; i++) {
-        int pos = qrand() % size;
-        swap(array, pos, i);
-    }
-}
-
 static bool isValidSlot(const QMetaMethod &sl)
 {
     if (sl.access() != QMetaMethod::Private || !sl.parameterTypes().isEmpty()
@@ -1046,9 +1012,6 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
          " -v1        : Print enter messages for each testfunction\n"
          " -v2        : Also print out each QVERIFY/QCOMPARE/QTEST\n"
          " -vs        : Print every signal emitted\n"
-         " -random    : Run testcases within each test in random order\n"
-         " -seed n    : Positive integer to be used as seed for -random. If not specified,\n"
-         "              the current time will be used as seed.\n"
          " -eventdelay ms    : Set default delay for mouse and keyboard simulation to ms milliseconds\n"
          " -keydelay ms      : Set default delay for keyboard simulation to ms milliseconds\n"
          " -mousedelay ms    : Set default delay for mouse simulation to ms milliseconds\n"
@@ -1177,22 +1140,6 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
 #endif
         } else if (strcmp(argv[i], "-eventcounter") == 0) {
             QBenchmarkGlobalData::current->setMode(QBenchmarkGlobalData::EventCounter);
-        } else if (strcmp(argv[i], "-random") == 0) {
-            QTest::randomOrder = true;
-        } else if (strcmp(argv[i], "-seed") == 0) {
-            bool argumentOk = false;
-            if (i + 1 < argc) {
-                char * endpt = 0;
-                long longSeed = strtol(argv[++i], &endpt, 10);
-                argumentOk = (*endpt == '\0' && longSeed >= 0);
-                QTest::seed = longSeed;
-            }
-            if (!argumentOk) {
-                fprintf(stderr, "-seed needs an extra positive integer parameter to specify the seed\n");
-                exit(1);
-            } else {
-                QTest::seedSet = true;
-            }
         } else if (strcmp(argv[i], "-minimumvalue") == 0) {
             if (i + 1 >= argc) {
                 fprintf(stderr, "-minimumvalue needs an extra parameter to indicate the minimum time(ms)\n");
@@ -1297,11 +1244,6 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
             QTEST_ASSERT(QTest::testFuncCount < 512);
         }
     }
-
-    if (QTest::seedSet && !QTest::randomOrder) {
-        fprintf(stderr, "-seed requires -random\n");
-        exit(1);
-    }
 }
 
 QBenchmarkResult qMedian(const QList<QBenchmarkResult> &container)
@@ -1596,11 +1538,7 @@ static void qInvokeTestMethods(QObject *testObject)
 {
     const QMetaObject *metaObject = testObject->metaObject();
     QTEST_ASSERT(metaObject);
-    if (QTest::randomOrder) {
-        QTestLog::startLogging(QTest::seed);
-    } else {
-        QTestLog::startLogging();
-    }
+    QTestLog::startLogging();
     QTestResult::setCurrentTestFunction("initTestCase");
     QTestResult::setCurrentTestLocation(QTestResult::DataFunc);
     QTestTable::globalTestTable();
@@ -1617,8 +1555,6 @@ static void qInvokeTestMethods(QObject *testObject)
         if(!QTestResult::skipCurrentTest() && !previousFailed) {
 
             if (QTest::testFuncs) {
-                if (QTest::randomOrder)
-                    randomizeList(QTest::testFuncs, QTest::testFuncCount);
                 for (int i = 0; i != QTest::testFuncCount; i++) {
                     if (!qInvokeTestMethod(metaObject->method(QTest::testFuncs[i].function()).signature(),
                                                               QTest::testFuncs[i].data())) {
@@ -1631,8 +1567,6 @@ static void qInvokeTestMethods(QObject *testObject)
                 QMetaMethod *testMethods = new QMetaMethod[methodCount];
                 for (int i = 0; i != methodCount; i++)
                     testMethods[i] = metaObject->method(i);
-                if (QTest::randomOrder)
-                    randomizeList(testMethods, methodCount);
                 for (int i = 0; i != methodCount; i++) {
                     if (!isValidSlot(testMethods[i]))
                         continue;
@@ -1840,9 +1774,6 @@ int QTest::qExec(QObject *testObject, int argc, char **argv)
 
     QTestResult::setCurrentTestObject(metaObject->className());
     qtest_qParseArgs(argc, argv, false);
-    if (QTest::randomOrder) {
-        seedRandom();
-    }
 #ifdef QTESTLIB_USE_VALGRIND
     if (QBenchmarkGlobalData::current->mode() == QBenchmarkGlobalData::CallgrindParentProcess) {
         const QStringList origAppArgs(QCoreApplication::arguments());
diff --git a/src/testlib/qtestlightxmlstreamer.cpp b/src/testlib/qtestlightxmlstreamer.cpp
index 8ac4e03d77..c0010cc522 100644
--- a/src/testlib/qtestlightxmlstreamer.cpp
+++ b/src/testlib/qtestlightxmlstreamer.cpp
@@ -165,13 +165,8 @@ void QTestLightXmlStreamer::formatBeforeAttributes(const QTestElement *element,
 void QTestLightXmlStreamer::output(QTestElement *element) const
 {
     QTestCharBuffer buf;
-    if (logger()->hasRandomSeed()) {
-        QTest::qt_asprintf(&buf, "<Environment>\n    <QtVersion>%s</QtVersion>\n    <QTestVersion>%s</QTestVersion>\n    <RandomSeed>%d</RandomSeed>\n",
-                       qVersion(), QTEST_VERSION_STR, logger()->randomSeed() );
-    } else {
-        QTest::qt_asprintf(&buf, "<Environment>\n    <QtVersion>%s</QtVersion>\n    <QTestVersion>%s</QTestVersion>\n",
+    QTest::qt_asprintf(&buf, "<Environment>\n    <QtVersion>%s</QtVersion>\n    <QTestVersion>%s</QTestVersion>\n",
                        qVersion(), QTEST_VERSION_STR );
-    }
     outputString(buf.constData());
 
     QTest::qt_asprintf(&buf, "</Environment>\n");
diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp
index 36b69e1155..0a87a4ce14 100644
--- a/src/testlib/qtestlog.cpp
+++ b/src/testlib/qtestlog.cpp
@@ -281,15 +281,6 @@ void QTestLog::addBenchmarkResult(const QBenchmarkResult &result)
     QTest::testLogger->addBenchmarkResult(result);
 }
 
-void QTestLog::startLogging(unsigned int randomSeed)
-{
-    QTEST_ASSERT(!QTest::testLogger);
-    QTest::initLogger();
-    QTest::testLogger->registerRandomSeed(randomSeed);
-    QTest::testLogger->startLogging(QTest::outFile);
-    QTest::oldMessageHandler = qInstallMsgHandler(QTest::messageHandler);
-}
-
 void QTestLog::startLogging()
 {
     QTEST_ASSERT(!QTest::testLogger);
diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h
index 006b3ac12f..9b580eb8cc 100644
--- a/src/testlib/qtestlog_p.h
+++ b/src/testlib/qtestlog_p.h
@@ -82,7 +82,6 @@ public:
     static void info(const char *msg, const char *file, int line);
 
     static void startLogging();
-    static void startLogging(unsigned int randomSeed);
     static void stopLogging();
 
     static void setLogMode(LogMode mode);
diff --git a/src/testlib/qtestlogger.cpp b/src/testlib/qtestlogger.cpp
index d53a968719..385fb866c2 100644
--- a/src/testlib/qtestlogger.cpp
+++ b/src/testlib/qtestlogger.cpp
@@ -62,8 +62,6 @@ QTestLogger::QTestLogger(int fm)
     , testCounter(0)
     , failureCounter(0)
     , errorCounter(0)
-    , randomSeed_(0)
-    , hasRandomSeed_(false)
 {
 }
 
@@ -128,14 +126,6 @@ void QTestLogger::stopLogging()
         property->addAttribute(QTest::AI_PropertyValue, qVersion());
         properties->addLogElement(property);
 
-        if (hasRandomSeed()) {
-            property = new QTestElement(QTest::LET_Property);
-            property->addAttribute(QTest::AI_Name, "RandomSeed");
-            QTest::qt_snprintf(buf, sizeof(buf), "%i", randomSeed());
-            property->addAttribute(QTest::AI_PropertyValue, buf);
-            properties->addLogElement(property);
-        }
-
         currentLogElement->addLogElement(properties);
 
         currentLogElement->addLogElement(iterator);
@@ -349,21 +339,5 @@ void QTestLogger::addMessage(MessageTypes type, const char *message, const char
     }
 }
 
-void QTestLogger::registerRandomSeed(unsigned int seed)
-{
-    randomSeed_ = seed;
-    hasRandomSeed_ = true;
-}
-
-unsigned int QTestLogger::randomSeed() const
-{
-    return randomSeed_;
-}
-
-bool QTestLogger::hasRandomSeed() const
-{
-    return hasRandomSeed_;
-}
-
 QT_END_NAMESPACE
 
diff --git a/src/testlib/qtestlogger_p.h b/src/testlib/qtestlogger_p.h
index eef5b92d2b..5e5b0f2f3f 100644
--- a/src/testlib/qtestlogger_p.h
+++ b/src/testlib/qtestlogger_p.h
@@ -87,10 +87,6 @@ class QTestLogger : public QAbstractTestLogger
         void addMessage(MessageTypes type, const char *message,
                     const char *file = 0, int line = 0);
 
-        void registerRandomSeed(unsigned int seed);
-        unsigned int randomSeed() const;
-        bool hasRandomSeed() const;
-
     private:
         QTestElement *listOfTestcases;
         QTestElement *currentLogElement;
@@ -101,8 +97,6 @@ class QTestLogger : public QAbstractTestLogger
         int testCounter;
         int failureCounter;
         int errorCounter;
-        unsigned int randomSeed_;
-        bool hasRandomSeed_;
 };
 
 QT_END_NAMESPACE
diff --git a/src/testlib/qtestxmlstreamer.cpp b/src/testlib/qtestxmlstreamer.cpp
index 7a4115262b..47c4463b5a 100644
--- a/src/testlib/qtestxmlstreamer.cpp
+++ b/src/testlib/qtestxmlstreamer.cpp
@@ -205,13 +205,8 @@ void QTestXmlStreamer::output(QTestElement *element) const
                        quotedTc.constData());
     outputString(buf.constData());
 
-    if (logger()->hasRandomSeed()) {
-        QTest::qt_asprintf(&buf, "<Environment>\n    <QtVersion>%s</QtVersion>\n    <QTestVersion>%s</QTestVersion>\n    <RandomSeed>%d</RandomSeed>\n",
-                           qVersion(), QTEST_VERSION_STR, logger()->randomSeed() );
-    } else {
-        QTest::qt_asprintf(&buf, "<Environment>\n    <QtVersion>%s</QtVersion>\n    <QTestVersion>%s</QTestVersion>\n",
-                           qVersion(), QTEST_VERSION_STR );
-    }
+    QTest::qt_asprintf(&buf, "<Environment>\n    <QtVersion>%s</QtVersion>\n    <QTestVersion>%s</QTestVersion>\n",
+                       qVersion(), QTEST_VERSION_STR );
     outputString(buf.constData());
 
     QTest::qt_asprintf(&buf, "</Environment>\n");
diff --git a/src/testlib/qxmltestlogger.cpp b/src/testlib/qxmltestlogger.cpp
index 15b5c84fd5..5ed7079ab6 100644
--- a/src/testlib/qxmltestlogger.cpp
+++ b/src/testlib/qxmltestlogger.cpp
@@ -93,9 +93,8 @@ namespace QTest {
 
 
 QXmlTestLogger::QXmlTestLogger(XmlMode mode )
-    :xmlmode(mode), randomSeed(0), hasRandomSeed(false)
+    : xmlmode(mode)
 {
-
 }
 
 QXmlTestLogger::~QXmlTestLogger()
@@ -116,20 +115,11 @@ void QXmlTestLogger::startLogging(const char *filename)
         outputString(buf.constData());
     }
 
-    if (hasRandomSeed) {
-       QTest::qt_asprintf(&buf,
-                "<Environment>\n"
-                "    <QtVersion>%s</QtVersion>\n"
-                "    <QTestVersion>"QTEST_VERSION_STR"</QTestVersion>\n"
-                "    <RandomSeed>%d</RandomSeed>\n"
-                "</Environment>\n", qVersion(), randomSeed);
-    } else {
-       QTest::qt_asprintf(&buf,
+   QTest::qt_asprintf(&buf,
                 "<Environment>\n"
                 "    <QtVersion>%s</QtVersion>\n"
                 "    <QTestVersion>"QTEST_VERSION_STR"</QTestVersion>\n"
                 "</Environment>\n", qVersion());
-    }
     outputString(buf.constData());
 }
 
@@ -450,10 +440,4 @@ int QXmlTestLogger::xmlCdata(QTestCharBuffer* str, char const* src)
     return allocateStringFn(str, src, QXmlTestLogger::xmlCdata);
 }
 
-void QXmlTestLogger::registerRandomSeed(unsigned int seed)
-{
-    randomSeed = seed;
-    hasRandomSeed = true;
-}
-
 QT_END_NAMESPACE
diff --git a/src/testlib/qxmltestlogger_p.h b/src/testlib/qxmltestlogger_p.h
index ad510d5ce0..c08f5e469e 100644
--- a/src/testlib/qxmltestlogger_p.h
+++ b/src/testlib/qxmltestlogger_p.h
@@ -79,8 +79,6 @@ public:
     void addMessage(MessageTypes type, const char *message,
                     const char *file = 0, int line = 0);
 
-    void registerRandomSeed(unsigned int seed);
-
     static int xmlCdata(QTestCharBuffer *dest, char const* src);
     static int xmlQuote(QTestCharBuffer *dest, char const* src);
     static int xmlCdata(QTestCharBuffer *dest, char const* src, size_t n);
@@ -88,8 +86,6 @@ public:
 
 private:
     XmlMode xmlmode;
-    unsigned int randomSeed;
-    bool hasRandomSeed;
 };
 
 QT_END_NAMESPACE