Refactor qscriptv8testsuite to make it more maintainable

Move the infrastructure for creating a dynamic test object
to AbstractTestSuite, so that it can be reused by the
qscriptjstestsuite test as well (change coming in next commit).

Introduce configuration files for defining skipped tests and
expected failures; this was previously embedded in the C++ code,
which made it hard to update.

Make it possible to override the default test locations through
environment variables. This makes it easy to run the autotest
against an external repository (e.g. WebKit or V8 trunk), and
even different revisions of those repositories.

Task-number: QTBUG-17903
Reviewed-by: Jedrzej Nowacki

8 files changed, 771 insertions, 223 deletions

diff --git a/tests/auto/qscriptv8testsuite/abstracttestsuite.cpp b/tests/auto/qscriptv8testsuite/abstracttestsuite.cpp
new file mode 100644
index 0000000000..d47eb2429b
--- /dev/null
+++ b/tests/auto/qscriptv8testsuite/abstracttestsuite.cpp
@@ -0,0 +1,481 @@
+/****************************************************************************
+**
+** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "abstracttestsuite.h"
+#include <QtTest/QtTest>
+#include <QtCore/qset.h>
+#include <QtCore/qtextstream.h>
+
+/*!
+   AbstractTestSuite provides a way of building QtTest test objects
+   dynamically. The use case is integration of JavaScript test suites
+   into QtTest autotests.
+
+   Subclasses add their tests functions with addTestFunction() in the
+   constructor, and must reimplement runTestFunction(). Additionally,
+   subclasses can reimplement initTestCase() and cleanupTestCase()
+   (but make sure to call the base implementation).
+
+   AbstractTestSuite uses configuration files for getting information
+   about skipped tests (skip.txt) and expected test failures
+   (expect_fail.txt). Subclasses must reimplement
+   createSkipConfigFile() and createExpectFailConfigFile() for
+   creating these files, and configData() for processing an entry of
+   such a file.
+
+   The config file format is as follows:
+   - Lines starting with '#' are skipped.
+   - Lines of the form [SYMBOL] means that the upcoming data
+     should only be processed if the given SYMBOL is defined on
+     this platform.
+   - Any other line is split on ' | ' and handed off to the client.
+
+   Subclasses must provide a default tests directory (where the
+   subclass expects to find the script files to run as tests), and a
+   default config file directory. Some environment variables can be
+   used to affect where AbstractTestSuite will look for files:
+
+   - QTSCRIPT_TEST_CONFIG_DIR: Overrides the default test config path.
+
+   - QTSCRIPT_TEST_CONFIG_SUFFIX: Is appended to "skip" and
+   "expect_fail" to create the test config name. This makes it easy to
+   maintain skip- and expect_fail-files corresponding to different
+   revisions of a test suite, and switch between them.
+
+   - QTSCRIPT_TEST_DIR: Overrides the default test dir.
+
+   AbstractTestSuite does _not_ define how the test dir itself is
+   processed or how tests are run; this is left up to the subclass.
+
+   If no config files are found, AbstractTestSuite will ask the
+   subclass to create a default skip file. Also, the
+   shouldGenerateExpectedFailures variable will be set to true. The
+   subclass should check for this when a test fails, and add an entry
+   to its set of expected failures. When all tests have been run,
+   AbstractTestSuite will ask the subclass to create the expect_fail
+   file based on the tests that failed. The next time the autotest is
+   run, the created config files will be used.
+
+   The reason for skipping a test is usually that it takes a very long
+   time to complete (or even hangs completely), or it crashes. It's
+   not possible for the test runner to know in advance which tests are
+   problematic, which is why the entries to the skip file are
+   typically added manually. When running tests for the first time, it
+   can be useful to run the autotest with the -v1 command line option,
+   so you can see the name of each test before it's run, and can add a
+   skip entry if appropriate.
+*/
+
+// Helper class for constructing the test class's QMetaObject contents
+// at runtime.
+class TestMetaObjectBuilder
+{
+public:
+    TestMetaObjectBuilder(const QByteArray &className,
+                      const QMetaObject *superClass);
+
+    void appendPrivateVoidSlot(const char *signature);
+    void appendPrivateVoidSlot(const QString &signature)
+    { appendPrivateVoidSlot(signature.toLatin1().constData()); }
+
+    void assignContents(QMetaObject &);
+
+private:
+    void appendString(const char *);
+    void finalize();
+
+    const QByteArray m_className;
+    const QMetaObject *m_superClass;
+    QVector<uint> m_data;
+    QVector<char> m_stringdata;
+    int m_emptyStringOffset;
+    bool m_finalized;
+};
+
+TestMetaObjectBuilder::TestMetaObjectBuilder(
+    const QByteArray &className,
+    const QMetaObject *superClass)
+    : m_className(className), m_superClass(superClass),
+      m_finalized(false)
+{
+    // header
+    m_data << 1 // revision
+          << 0 // classname
+          << 0 << 0 // classinfo
+          << 0 << 10 // methods (backpatched later)
+          << 0 << 0 // properties
+          << 0 << 0 // enums/sets
+        ;
+
+    appendString(className.constData());
+    m_emptyStringOffset = m_stringdata.size();
+    appendString("");
+}
+
+void TestMetaObjectBuilder::appendString(const char *s)
+{
+    char c;
+    do {
+        c = *(s++);
+        m_stringdata << c;
+    } while (c != '\0');
+}
+
+void TestMetaObjectBuilder::appendPrivateVoidSlot(const char *signature)
+{
+    static const int methodCountOffset = 4;
+    // signature, parameters, type, tag, flags
+    m_data << m_stringdata.size()
+           << m_emptyStringOffset
+           << m_emptyStringOffset
+           << m_emptyStringOffset
+           << 0x08;
+    appendString(signature);
+    ++m_data[methodCountOffset];
+}
+
+void TestMetaObjectBuilder::finalize()
+{
+    if (m_finalized)
+        return;
+    m_data << 0; // eod
+    m_finalized = true;
+}
+
+/**
+  Assigns this builder's contents to the meta-object \a mo.  It's up
+  to the caller to ensure that this builder (and hence, its data)
+  stays alive as long as needed.
+*/
+void TestMetaObjectBuilder::assignContents(QMetaObject &mo)
+{
+    finalize();
+    mo.d.superdata = m_superClass;
+    mo.d.stringdata = m_stringdata.constData();
+    mo.d.data = m_data.constData();
+    mo.d.extradata = 0;
+}
+
+
+class TestConfigClientInterface;
+// For parsing information about skipped tests and expected failures.
+class TestConfigParser
+{
+public:
+    static void parse(const QString &path,
+                      TestConfig::Mode mode,
+                      TestConfigClientInterface *client);
+
+private:
+    static QString unescape(const QString &);
+    static bool isKnownSymbol(const QString &);
+    static bool isDefined(const QString &);
+
+    static QSet<QString> knownSymbols;
+    static QSet<QString> definedSymbols;
+};
+
+QSet<QString> TestConfigParser::knownSymbols;
+QSet<QString> TestConfigParser::definedSymbols;
+
+/**
+   Parses the config file at the given \a path in the given \a mode.
+   Handling of errors and data is delegated to the given \a client.
+*/
+void TestConfigParser::parse(const QString &path,
+                             TestConfig::Mode mode,
+                             TestConfigClientInterface *client)
+{
+    QFile file(path);
+    if (!file.open(QIODevice::ReadOnly))
+        return;
+    QTextStream stream(&file);
+    int lineNumber = 0;
+    QString predicate;
+    const QString separator = QString::fromLatin1(" | ");
+    while (!stream.atEnd()) {
+        ++lineNumber;
+        QString line = stream.readLine();
+        if (line.isEmpty())
+            continue;
+        if (line.startsWith('#')) // Comment
+            continue;
+        if (line.startsWith('[')) { // Predicate
+            if (!line.endsWith(']')) {
+                client->configError(path, "malformed predicate", lineNumber);
+                return;
+            }
+            QString symbol = line.mid(1, line.size()-2);
+            if (isKnownSymbol(symbol)) {
+                predicate = symbol;
+            } else {
+                qWarning("symbol %s is not known -- add it to TestConfigParser!", qPrintable(symbol));
+                predicate = QString();
+            }
+        } else {
+            if (predicate.isEmpty() || isDefined(predicate)) {
+                QStringList parts = line.split(separator, QString::KeepEmptyParts);
+                for (int i = 0; i < parts.size(); ++i)
+                    parts[i] = unescape(parts[i]);
+                client->configData(mode, parts);
+            }
+        }
+    }
+}
+
+QString TestConfigParser::unescape(const QString &str)
+{
+    return QString(str).replace("\\n", "\n");
+}
+
+bool TestConfigParser::isKnownSymbol(const QString &symbol)
+{
+    if (knownSymbols.isEmpty()) {
+        knownSymbols
+            // If you add a symbol here, add a case for it in
+            // isDefined() as well.
+            << "Q_OS_LINUX"
+            << "Q_OS_SOLARIS"
+            << "Q_OS_WINCE"
+            << "Q_OS_SYMBIAN"
+            << "Q_CC_MSVC"
+            << "Q_CC_MINGW"
+            ;
+    }
+    return knownSymbols.contains(symbol);
+}
+
+bool TestConfigParser::isDefined(const QString &symbol)
+{
+    if (definedSymbols.isEmpty()) {
+        definedSymbols
+#ifdef Q_OS_LINUX
+            << "Q_OS_LINUX"
+#endif
+#ifdef Q_OS_SOLARIS
+            << "Q_OS_SOLARIS"
+#endif
+#ifdef Q_OS_WINCE
+            << "Q_OS_WINCE"
+#endif
+#ifdef Q_OS_SYMBIAN
+            << "Q_OS_SYMBIAN"
+#endif
+#ifdef Q_CC_MSVC
+            << "Q_CC_MSVC"
+#endif
+#ifdef Q_CC_MINGW
+            << "Q_CC_MINGW"
+#endif
+            ;
+    }
+    return definedSymbols.contains(symbol);
+}
+
+
+QMetaObject AbstractTestSuite::staticMetaObject;
+
+const QMetaObject *AbstractTestSuite::metaObject() const
+{
+    return &staticMetaObject;
+}
+
+void *AbstractTestSuite::qt_metacast(const char *_clname)
+{
+    if (!_clname) return 0;
+    if (!strcmp(_clname, staticMetaObject.d.stringdata))
+        return static_cast<void*>(const_cast<AbstractTestSuite*>(this));
+    return QObject::qt_metacast(_clname);
+}
+
+int AbstractTestSuite::qt_metacall(QMetaObject::Call _c, int _id, void **_a)
+{
+    _id = QObject::qt_metacall(_c, _id, _a);
+    if (_id < 0)
+        return _id;
+    if (_c == QMetaObject::InvokeMetaMethod) {
+        switch (_id) {
+        case 0:
+            initTestCase();
+            break;
+        case 1:
+            cleanupTestCase();
+            break;
+        default:
+            // If another method is added above, this offset must be adjusted.
+            runTestFunction(_id - 2);
+        }
+        _id -= staticMetaObject.methodCount() - staticMetaObject.methodOffset();
+    }
+    return _id;
+}
+
+AbstractTestSuite::AbstractTestSuite(const QByteArray &className,
+                                     const QString &defaultTestsPath,
+                                     const QString &defaultConfigPath)
+    : shouldGenerateExpectedFailures(false),
+      metaBuilder(new TestMetaObjectBuilder(className, &QObject::staticMetaObject))
+{
+    QString testConfigPath = qgetenv("QTSCRIPT_TEST_CONFIG_DIR");
+    if (testConfigPath.isEmpty())
+        testConfigPath = defaultConfigPath;
+    QString configSuffix = qgetenv("QTSCRIPT_TEST_CONFIG_SUFFIX");
+    skipConfigPath = QString::fromLatin1("%0/skip%1.txt")
+                     .arg(testConfigPath).arg(configSuffix);
+    expectFailConfigPath = QString::fromLatin1("%0/expect_fail%1.txt")
+                           .arg(testConfigPath).arg(configSuffix);
+
+    QString testsPath = qgetenv("QTSCRIPT_TEST_DIR");
+    if (testsPath.isEmpty())
+        testsPath = defaultTestsPath;
+    testsDir = QDir(testsPath);
+
+    addTestFunction("initTestCase");
+    addTestFunction("cleanupTestCase");
+
+    // Subclass constructors should add their custom test functions to
+    // the meta-object and call finalizeMetaObject().
+}
+
+AbstractTestSuite::~AbstractTestSuite()
+{
+    delete metaBuilder;
+}
+
+void AbstractTestSuite::addTestFunction(const QString &name,
+                                        DataFunctionCreation dfc)
+{
+    if (dfc == CreateDataFunction) {
+        QString dataSignature = QString::fromLatin1("%0_data()").arg(name);
+        metaBuilder->appendPrivateVoidSlot(dataSignature);
+    }
+    QString signature = QString::fromLatin1("%0()").arg(name);
+    metaBuilder->appendPrivateVoidSlot(signature);
+}
+
+void AbstractTestSuite::finalizeMetaObject()
+{
+    metaBuilder->assignContents(staticMetaObject);
+}
+
+void AbstractTestSuite::initTestCase()
+{
+    if (!testsDir.exists()) {
+        QString message = QString::fromLatin1("tests directory (%0) doesn't exist.")
+                          .arg(testsDir.path());
+        QFAIL(qPrintable(message));
+        return;
+    }
+
+    if (QFileInfo(skipConfigPath).exists())
+        TestConfigParser::parse(skipConfigPath, TestConfig::Skip, this);
+    else
+        createSkipConfigFile();
+
+    if (QFileInfo(expectFailConfigPath).exists())
+        TestConfigParser::parse(expectFailConfigPath, TestConfig::ExpectFail, this);
+    else
+        shouldGenerateExpectedFailures = true;
+}
+
+void AbstractTestSuite::cleanupTestCase()
+{
+    if (shouldGenerateExpectedFailures)
+        createExpectFailConfigFile();
+}
+
+void AbstractTestSuite::configError(const QString &path, const QString &message, int lineNumber)
+{
+    QString output;
+    output.append(path);
+    if (lineNumber != -1)
+        output.append(":").append(QString::number(lineNumber));
+    output.append(": ").append(message);
+    QFAIL(qPrintable(output));
+}
+
+void AbstractTestSuite::createSkipConfigFile()
+{
+    QFile file(skipConfigPath);
+    if (!file.open(QIODevice::WriteOnly))
+        return;
+    QWARN(qPrintable(QString::fromLatin1("creating %0").arg(skipConfigPath)));
+    QTextStream stream(&file);
+
+    writeSkipConfigFile(stream);
+
+    file.close();
+}
+
+void AbstractTestSuite::createExpectFailConfigFile()
+{
+    QFile file(expectFailConfigPath);
+    if (!file.open(QFile::WriteOnly))
+        return;
+    QWARN(qPrintable(QString::fromLatin1("creating %0").arg(expectFailConfigPath)));
+    QTextStream stream(&file);
+
+    writeExpectFailConfigFile(stream);
+
+    file.close();
+}
+
+/*!
+  Convenience function for reading all contents of a file.
+ */
+QString AbstractTestSuite::readFile(const QString &filename)
+{
+    QFile file(filename);
+    if (!file.open(QFile::ReadOnly))
+        return QString();
+    QTextStream stream(&file);
+    stream.setCodec("UTF-8");
+    return stream.readAll();
+}
+
+/*!
+  Escapes characters in the string \a str so it's suitable for writing
+  to a config file.
+ */
+QString AbstractTestSuite::escape(const QString &str)
+{
+    return QString(str).replace("\n", "\\n");
+}
diff --git a/tests/auto/qscriptv8testsuite/abstracttestsuite.h b/tests/auto/qscriptv8testsuite/abstracttestsuite.h
new file mode 100644
index 0000000000..ee9e251379
--- /dev/null
+++ b/tests/auto/qscriptv8testsuite/abstracttestsuite.h
@@ -0,0 +1,125 @@
+/****************************************************************************
+**
+** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef ABSTRACTTESTSUITE_H
+#define ABSTRACTTESTSUITE_H
+
+#include <QtCore/qobject.h>
+
+#include <QtCore/qbytearray.h>
+#include <QtCore/qdir.h>
+#include <QtCore/qstring.h>
+#include <QtCore/qstringlist.h>
+#include <QtCore/qvector.h>
+
+class QTextStream;
+class TestMetaObjectBuilder;
+
+namespace TestConfig {
+enum Mode {
+    Skip,
+    ExpectFail
+};
+}
+
+// For receiving callbacks from the config parser.
+class TestConfigClientInterface
+{
+public:
+    virtual ~TestConfigClientInterface() {}
+    virtual void configData(TestConfig::Mode mode,
+                            const QStringList &parts) = 0;
+    virtual void configError(const QString &path,
+                             const QString &message,
+                             int lineNumber) = 0;
+};
+
+class AbstractTestSuite : public QObject,
+                          public TestConfigClientInterface
+{
+// No Q_OBJECT macro, we implement the meta-object ourselves.
+public:
+    AbstractTestSuite(const QByteArray &className,
+                      const QString &defaultTestsPath,
+                      const QString &defaultConfigPath);
+    virtual ~AbstractTestSuite();
+
+    static QMetaObject staticMetaObject;
+    virtual const QMetaObject *metaObject() const;
+    virtual void *qt_metacast(const char *);
+    virtual int qt_metacall(QMetaObject::Call, int, void **argv);
+
+    static QString readFile(const QString &);
+    static QString escape(const QString &);
+
+protected:
+    enum DataFunctionCreation {
+        DontCreateDataFunction,
+        CreateDataFunction
+    };
+
+    void addTestFunction(const QString &,
+                         DataFunctionCreation = DontCreateDataFunction);
+    void finalizeMetaObject();
+
+    virtual void initTestCase();
+    virtual void cleanupTestCase();
+
+    virtual void writeSkipConfigFile(QTextStream &) = 0;
+    virtual void writeExpectFailConfigFile(QTextStream &) = 0;
+
+    virtual void runTestFunction(int index) = 0;
+
+    virtual void configError(const QString &path, const QString &message, int lineNumber);
+
+    QDir testsDir;
+    bool shouldGenerateExpectedFailures;
+
+private:
+    TestMetaObjectBuilder *metaBuilder;
+    QString skipConfigPath, expectFailConfigPath;
+
+private:
+    void createSkipConfigFile();
+    void createExpectFailConfigFile();
+};
+
+#endif
diff --git a/tests/auto/qscriptv8testsuite/abstracttestsuite.pri b/tests/auto/qscriptv8testsuite/abstracttestsuite.pri
new file mode 100644
index 0000000000..1de5b9398b
--- /dev/null
+++ b/tests/auto/qscriptv8testsuite/abstracttestsuite.pri
@@ -0,0 +1,4 @@
+SOURCES += $$PWD/abstracttestsuite.cpp
+HEADERS += $$PWD/abstracttestsuite.h
+INCLUDEPATH += $$PWD
+DEPENDPATH += $$PWD
diff --git a/tests/auto/qscriptv8testsuite/expect_fail.txt b/tests/auto/qscriptv8testsuite/expect_fail.txt
new file mode 100644
index 0000000000..a4eee73be2
--- /dev/null
+++ b/tests/auto/qscriptv8testsuite/expect_fail.txt
@@ -0,0 +1,16 @@
+# testcase | actual | expected | message
+arguments-enum | 2 | 0
+const-redecl | undefined | TypeError | local:'const x; var x'
+date-parse | NaN | 946713600000 | Sat, 01-Jan-2000 08:00:00 GMT+00:00
+delete-global-properties | true | false
+delete | false | true | delete 100
+function-arguments-null | false | true
+function-caller | null | function eval() {\n    [native code]\n}
+function-prototype | prototype | disconnectconnect
+global-const-var-conflicts | false | true
+number-tostring | 0 | 0.0000a7c5ac471b4788
+parse-int-float | 1e+21 | 1
+regexp | false | true
+string-lastindexof | 0 | -1
+string-split | 4 | 3 | 19 - array length
+substr | abcdefghijklmn | 
diff --git a/tests/auto/qscriptv8testsuite/qscriptv8testsuite.pro b/tests/auto/qscriptv8testsuite/qscriptv8testsuite.pro
index 00e2e01fc6..e1c623467f 100644
--- a/tests/auto/qscriptv8testsuite/qscriptv8testsuite.pro
+++ b/tests/auto/qscriptv8testsuite/qscriptv8testsuite.pro
@@ -2,3 +2,4 @@ load(qttest_p4)
 QT = core script
 SOURCES  += tst_qscriptv8testsuite.cpp
 RESOURCES += qscriptv8testsuite.qrc
+include(abstracttestsuite.pri)
diff --git a/tests/auto/qscriptv8testsuite/qscriptv8testsuite.qrc b/tests/auto/qscriptv8testsuite/qscriptv8testsuite.qrc
index a894ee5859..150ccf0a0a 100644
--- a/tests/auto/qscriptv8testsuite/qscriptv8testsuite.qrc
+++ b/tests/auto/qscriptv8testsuite/qscriptv8testsuite.qrc
@@ -1,5 +1,7 @@
 <!DOCTYPE RCC><RCC version="1.0">
 <qresource>
     <file>tests</file>
+    <file>expect_fail.txt</file>
+    <file>skip.txt</file>
 </qresource>
 </RCC>
diff --git a/tests/auto/qscriptv8testsuite/skip.txt b/tests/auto/qscriptv8testsuite/skip.txt
new file mode 100644
index 0000000000..3c2cc53fa4
--- /dev/null
+++ b/tests/auto/qscriptv8testsuite/skip.txt
@@ -0,0 +1,17 @@
+# testcase | message
+debug-* | not applicable
+mirror-* | not applicable
+array-concat | Hangs on JSC backend
+array-splice | Hangs on JSC backend
+sparse-array-reverse | Hangs on JSC backend
+string-case | V8-specific behavior? (Doesn't pass on SpiderMonkey either)
+
+[Q_OS_WINCE]
+deep-recursion | Demands too much memory on WinCE
+nested-repetition-count-overflow | Demands too much memory on WinCE
+unicode-test | Demands too much memory on WinCE
+mul-exhaustive | Demands too much memory on WinCE
+
+[Q_OS_SYMBIAN]
+nested-repetition-count-overflow | Demands too much memory on Symbian
+unicode-test | Demands too much memory on Symbian
diff --git a/tests/auto/qscriptv8testsuite/tst_qscriptv8testsuite.cpp b/tests/auto/qscriptv8testsuite/tst_qscriptv8testsuite.cpp
index 7d0858e49d..cf86c4d588 100644
--- a/tests/auto/qscriptv8testsuite/tst_qscriptv8testsuite.cpp
+++ b/tests/auto/qscriptv8testsuite/tst_qscriptv8testsuite.cpp
@@ -40,68 +40,33 @@
 ****************************************************************************/
 
 
+#include "abstracttestsuite.h"
 #include <QtTest/QtTest>
-#include <QByteArray>
-
 #include <QtScript>
 
 //TESTED_CLASS=
 //TESTED_FILES=
 
-// Uncomment the following define to have the autotest generate
-// addExpectedFailure() code for all the tests that fail.
-// This is useful when a whole new test (sub)suite is added.
-// The code is stored in addexpectedfailures.cpp.
-// Paste the contents into this file after the existing
-// addExpectedFailure() calls.
-
-//#define GENERATE_ADDEXPECTEDFAILURE_CODE
-
-static QString readFile(const QString &filename)
-{
-    QFile file(filename);
-    if (!file.open(QFile::ReadOnly))
-        return QString();
-    QTextStream stream(&file);
-    stream.setCodec("UTF-8");
-    return stream.readAll();
-}
-
-static void appendCString(QVector<char> *v, const char *s)
+class tst_Suite : public AbstractTestSuite
 {
-    char c;
-    do {
-        c = *(s++);
-        *v << c;
-    } while (c != '\0');
-}
-
-struct ExpectedFailure
-{
-    ExpectedFailure(const QString &name, const QString &act,
-                    const QString &exp, const QString &msg)
-        : testName(name), actual(act), expected(exp), message(msg)
-        { }
-
-    QString testName;
-    QString actual;
-    QString expected;
-    QString message;
-};
-
-class tst_Suite : public QObject
-{
-
 public:
     tst_Suite();
     virtual ~tst_Suite();
 
-    static QMetaObject staticMetaObject;
-    virtual const QMetaObject *metaObject() const;
-    virtual void *qt_metacast(const char *);
-    virtual int qt_metacall(QMetaObject::Call, int, void **argv);
+protected:
+    struct ExpectedFailure
+    {
+        ExpectedFailure(const QString &name, const QString &act,
+                        const QString &exp, const QString &msg)
+            : testName(name), actual(act), expected(exp), message(msg)
+            { }
+
+        QString testName;
+        QString actual;
+        QString expected;
+        QString message;
+    };
 
-private:
     void addExpectedFailure(const QString &testName, const QString &actual,
                             const QString &expected, const QString &message);
     bool isExpectedFailure(const QString &testName, const QString &actual,
@@ -110,34 +75,23 @@ private:
     void addTestExclusion(const QRegExp &rx, const QString &message);
     bool isExcludedTest(const QString &testName, QString *message) const;
 
-    QDir testsDir;
+    virtual void initTestCase();
+    virtual void configData(TestConfig::Mode mode, const QStringList &parts);
+    virtual void writeSkipConfigFile(QTextStream &);
+    virtual void writeExpectFailConfigFile(QTextStream &);
+    virtual void runTestFunction(int testIndex);
+
     QStringList testNames;
     QList<ExpectedFailure> expectedFailures;
     QList<QPair<QRegExp, QString> > testExclusions;
     QString mjsunitContents;
-#ifdef GENERATE_ADDEXPECTEDFAILURE_CODE
-    QString generatedAddExpectedFailureCode;
-#endif
 };
 
-QMetaObject tst_Suite::staticMetaObject;
-
-Q_GLOBAL_STATIC(QVector<uint>, qt_meta_data_tst_Suite)
-Q_GLOBAL_STATIC(QVector<char>, qt_meta_stringdata_tst_Suite)
-
-const QMetaObject *tst_Suite::metaObject() const
-{
-    return &staticMetaObject;
-}
-
-void *tst_Suite::qt_metacast(const char *_clname)
-{
-    if (!_clname) return 0;
-    if (!strcmp(_clname, qt_meta_stringdata_tst_Suite()->constData()))
-        return static_cast<void*>(const_cast<tst_Suite*>(this));
-    return QObject::qt_metacast(_clname);
-}
-
+// We expect failing tests to call the fail() function (defined in
+// mjsunit.js) with arguments expected, actual, message_opt.  This
+// function intercepts the call, calls the real fail() function (which
+// will throw an exception), and sets the original arguments on the
+// exception object so that we can process them later.
 static QScriptValue qscript_fail(QScriptContext *ctx, QScriptEngine *eng)
 {
     QScriptValue realFail = ctx->callee().data();
@@ -146,184 +100,132 @@ static QScriptValue qscript_fail(QScriptContext *ctx, QScriptEngine *eng)
     Q_ASSERT(eng->hasUncaughtException());
     ret.setProperty("expected", ctx->argument(0));
     ret.setProperty("actual", ctx->argument(1));
+    ret.setProperty("message", ctx->argument(2));
     QScriptContextInfo info(ctx->parentContext()->parentContext());
     ret.setProperty("lineNumber", info.lineNumber());
     return ret;
 }
 
-int tst_Suite::qt_metacall(QMetaObject::Call _c, int _id, void **_a)
+void tst_Suite::writeSkipConfigFile(QTextStream &stream)
 {
-    _id = QObject::qt_metacall(_c, _id, _a);
-    if (_id < 0)
-        return _id;
-    if (_c == QMetaObject::InvokeMetaMethod) {
-        QString name = testNames.at(_id);
-        QString path = testsDir.absoluteFilePath(name + ".js");
-        QString excludeMessage;
-        if (isExcludedTest(name, &excludeMessage)) {
-            QTest::qSkip(excludeMessage.toLatin1(), QTest::SkipAll, path.toLatin1(), -1);
-        } else {
-            QScriptEngine engine;
-            engine.evaluate(mjsunitContents).toString();
-            if (engine.hasUncaughtException()) {
-                QStringList bt = engine.uncaughtExceptionBacktrace();
-                QString err = engine.uncaughtException().toString();
-                qWarning("%s\n%s", qPrintable(err), qPrintable(bt.join("\n")));
-            } else {
-                QScriptValue fakeFail = engine.newFunction(qscript_fail);
-                fakeFail.setData(engine.globalObject().property("fail"));
-                engine.globalObject().setProperty("fail", fakeFail);
-                QString contents = readFile(path);
-                QScriptValue ret = engine.evaluate(contents);
-                if (engine.hasUncaughtException()) {
-                    if (!ret.isError()) {
-                        Q_ASSERT(ret.instanceOf(engine.globalObject().property("MjsUnitAssertionError")));
-                        QString actual = ret.property("actual").toString();
-                        QString expected = ret.property("expected").toString();
-                        int lineNumber = ret.property("lineNumber").toInt32();
-                        QString failMessage;
-                        if (isExpectedFailure(name, actual, expected, &failMessage)) {
-                            QTest::qExpectFail("", failMessage.toLatin1(),
-                                               QTest::Continue, path.toLatin1(),
-                                               lineNumber);
-                        }
-#ifdef GENERATE_ADDEXPECTEDFAILURE_CODE
-                        else {
-                            generatedAddExpectedFailureCode.append(
-                                "    addExpectedFailure(\"" + name
-                                + "\", \"" + actual + "\", \"" + expected
-                                + "\", willFixInNextReleaseMessage);\n");
-                        }
-#endif
-                        QTest::qCompare(actual, expected, "actual", "expect",
-                                        path.toLatin1(), lineNumber);
-                    } else {
-                        int lineNumber = ret.property("lineNumber").toInt32();
-                        QTest::qExpectFail("", ret.toString().toLatin1(),
-                                           QTest::Continue, path.toLatin1(), lineNumber);
-                        QTest::qVerify(false, ret.toString().toLatin1(), "", path.toLatin1(), lineNumber);
-                    }
-                }
-            }
-        }
-        _id -= testNames.size();
-    }
-    return _id;
+    stream << QString::fromLatin1("# testcase | message") << endl;
 }
 
-tst_Suite::tst_Suite()
+void tst_Suite::writeExpectFailConfigFile(QTextStream &stream)
 {
-    testsDir = QDir(":/tests");
-    if (!testsDir.exists()) {
-        qWarning("*** no tests/ dir!");
-    } else {
-        if (!testsDir.exists("mjsunit.js"))
-            qWarning("*** no tests/mjsunit.js file!");
-        else {
-            mjsunitContents = readFile(testsDir.absoluteFilePath("mjsunit.js"));
-            if (mjsunitContents.isEmpty())
-                qWarning("*** tests/mjsunit.js is empty!");
-        }
+    stream << QString::fromLatin1("# testcase | actual | expected | message") << endl;
+    for (int i = 0; i < expectedFailures.size(); ++i) {
+        const ExpectedFailure &fail = expectedFailures.at(i);
+        stream << QString::fromLatin1("%0 | %1 | %2")
+            .arg(fail.testName)
+            .arg(escape(fail.actual))
+            .arg(escape(fail.expected));
+        if (!fail.message.isEmpty())
+            stream << QString::fromLatin1(" | %0").arg(escape(fail.message));
+        stream << endl;
     }
-    QString willFixInNextReleaseMessage = QString::fromLatin1("Will fix in next release");
-    addExpectedFailure("arguments-enum", "2", "0", willFixInNextReleaseMessage);
-    addExpectedFailure("const-redecl", "undefined", "TypeError", willFixInNextReleaseMessage);
-    addExpectedFailure("global-const-var-conflicts", "false", "true", willFixInNextReleaseMessage);
-    addExpectedFailure("string-lastindexof", "0", "-1", "test is wrong?");
-
-#ifndef Q_OS_LINUX
-    addExpectedFailure("to-precision", "1.235e+27", "1.234e+27", "QTBUG-8053: toPrecision(4) gives wrong result on Mac");
-#endif
-
-#ifdef Q_OS_SOLARIS
-    addExpectedFailure("math-min-max", "Infinity", "-Infinity", willFixInNextReleaseMessage);
-    addExpectedFailure("negate-zero", "false", "true", willFixInNextReleaseMessage);
-    addExpectedFailure("str-to-num", "Infinity", "-Infinity", willFixInNextReleaseMessage);
-#endif
-
-    addTestExclusion("debug-*", "not applicable");
-    addTestExclusion("mirror-*", "not applicable");
-
-    addTestExclusion("array-concat", "Hangs on JSC backend");
-    addTestExclusion("array-splice", "Hangs on JSC backend");
-    addTestExclusion("sparse-array-reverse", "Hangs on JSC backend");
-
-    addTestExclusion("string-case", "V8-specific behavior? (Doesn't pass on SpiderMonkey either)");
-
-#ifdef Q_OS_WINCE
-    addTestExclusion("deep-recursion", "Demands too much memory on WinCE");
-    addTestExclusion("nested-repetition-count-overflow", "Demands too much memory on WinCE");
-    addTestExclusion("unicode-test", "Demands too much memory on WinCE");
-    addTestExclusion("mul-exhaustive", "Demands too much memory on WinCE");
-#endif
-
-#ifdef Q_OS_SYMBIAN
-    addTestExclusion("nested-repetition-count-overflow", "Demands too much memory on Symbian");
-    addTestExclusion("unicode-test", "Demands too much memory on Symbian");
-#endif
-    // Failures due to switch to JSC as back-end
-    addExpectedFailure("date-parse", "NaN", "946713600000", willFixInNextReleaseMessage);
-    addExpectedFailure("delete-global-properties", "true", "false", willFixInNextReleaseMessage);
-    addExpectedFailure("delete", "false", "true", willFixInNextReleaseMessage);
-    addExpectedFailure("function-arguments-null", "false", "true", willFixInNextReleaseMessage);
-    addExpectedFailure("function-caller", "null", "function eval() {\n    [native code]\n}", willFixInNextReleaseMessage);
-    addExpectedFailure("function-prototype", "prototype", "disconnectconnect", willFixInNextReleaseMessage);
-    addExpectedFailure("number-tostring", "0", "0.0000a7c5ac471b4788", willFixInNextReleaseMessage);
-    addExpectedFailure("parse-int-float", "1e+21", "1", willFixInNextReleaseMessage);
-    addExpectedFailure("regexp", "false", "true", willFixInNextReleaseMessage);
-    addExpectedFailure("smi-negative-zero", "-Infinity", "Infinity", willFixInNextReleaseMessage);
-    addExpectedFailure("string-split", "4", "3", willFixInNextReleaseMessage);
-    addExpectedFailure("substr", "abcdefghijklmn", "", willFixInNextReleaseMessage);
+}
 
-    static const char klass[] = "tst_QScriptV8TestSuite";
+void tst_Suite::runTestFunction(int testIndex)
+{
+    QString name = testNames.at(testIndex);
+    QString path = testsDir.absoluteFilePath(name + ".js");
 
-    QVector<uint> *data = qt_meta_data_tst_Suite();
-    // content:
-    *data << 1 // revision
-          << 0 // classname
-          << 0 << 0 // classinfo
-          << 0 << 10 // methods (backpatched later)
-          << 0 << 0 // properties
-          << 0 << 0 // enums/sets
-        ;
+    QString excludeMessage;
+    if (isExcludedTest(name, &excludeMessage)) {
+        QTest::qSkip(excludeMessage.toLatin1(), QTest::SkipAll, path.toLatin1(), -1);
+        return;
+    }
 
-    QVector<char> *stringdata = qt_meta_stringdata_tst_Suite();
-    appendCString(stringdata, klass);
-    appendCString(stringdata, "");
+    QScriptEngine engine;
+    engine.evaluate(mjsunitContents);
+    if (engine.hasUncaughtException()) {
+        QStringList bt = engine.uncaughtExceptionBacktrace();
+        QString err = engine.uncaughtException().toString();
+        qWarning("%s\n%s", qPrintable(err), qPrintable(bt.join("\n")));
+    } else {
+        // Prepare to intercept calls to mjsunit's fail() function.
+        QScriptValue fakeFail = engine.newFunction(qscript_fail);
+        fakeFail.setData(engine.globalObject().property("fail"));
+        engine.globalObject().setProperty("fail", fakeFail);
+
+        QString contents = readFile(path);
+        QScriptValue ret = engine.evaluate(contents);
+        if (engine.hasUncaughtException()) {
+            if (!ret.isError()) {
+                Q_ASSERT(ret.instanceOf(engine.globalObject().property("MjsUnitAssertionError")));
+                QString actual = ret.property("actual").toString();
+                QString expected = ret.property("expected").toString();
+                int lineNumber = ret.property("lineNumber").toInt32();
+                QString failMessage;
+                if (shouldGenerateExpectedFailures) {
+                    if (ret.property("message").isString())
+                        failMessage = ret.property("message").toString();
+                    addExpectedFailure(name, actual, expected, failMessage);
+                } else if (isExpectedFailure(name, actual, expected, &failMessage)) {
+                    QTest::qExpectFail("", failMessage.toLatin1(),
+                                       QTest::Continue, path.toLatin1(),
+                                       lineNumber);
+                }
+                QTest::qCompare(actual, expected, "actual", "expect",
+                                path.toLatin1(), lineNumber);
+            } else {
+                int lineNumber = ret.property("lineNumber").toInt32();
+                QTest::qExpectFail("", ret.toString().toLatin1(),
+                                   QTest::Continue, path.toLatin1(), lineNumber);
+                QTest::qVerify(false, ret.toString().toLatin1(), "", path.toLatin1(), lineNumber);
+            }
+        }
+    }
+}
 
+tst_Suite::tst_Suite()
+    : AbstractTestSuite("tst_QScriptV8TestSuite",
+                        ":/tests", ":/")
+{
+    // One test function per test file.
     QFileInfoList testFileInfos;
     testFileInfos = testsDir.entryInfoList(QStringList() << "*.js", QDir::Files);
     foreach (QFileInfo tfi, testFileInfos) {
         QString name = tfi.baseName();
-        // slot: signature, parameters, type, tag, flags
-        QString slot = QString::fromLatin1("%0()").arg(name);
-        static const int nullbyte = sizeof(klass);
-        *data << stringdata->size() << nullbyte << nullbyte << nullbyte << 0x08;
-        appendCString(stringdata, slot.toLatin1());
+        addTestFunction(name);
         testNames.append(name);
     }
 
-    (*data)[4] = testFileInfos.size();
+    finalizeMetaObject();
+}
 
-    *data << 0; // eod
+tst_Suite::~tst_Suite()
+{
+}
 
-    // initialize staticMetaObject
-    staticMetaObject.d.superdata = &QObject::staticMetaObject;
-    staticMetaObject.d.stringdata = stringdata->constData();
-    staticMetaObject.d.data = data->constData();
-    staticMetaObject.d.extradata = 0;
+void tst_Suite::initTestCase()
+{
+    AbstractTestSuite::initTestCase();
+
+    // FIXME: These warnings should be QFAIL, but that would make the
+    // test fail right now.
+    if (!testsDir.exists("mjsunit.js"))
+        qWarning("*** no tests/mjsunit.js file!");
+    else {
+        mjsunitContents = readFile(testsDir.absoluteFilePath("mjsunit.js"));
+        if (mjsunitContents.isEmpty())
+            qWarning("*** tests/mjsunit.js is empty!");
+    }
 }
 
-tst_Suite::~tst_Suite()
+void tst_Suite::configData(TestConfig::Mode mode, const QStringList &parts)
 {
-#ifdef GENERATE_ADDEXPECTEDFAILURE_CODE
-    if (!generatedAddExpectedFailureCode.isEmpty()) {
-        QFile file("addexpectedfailures.cpp");
-        file.open(QFile::WriteOnly);
-        QTextStream ts(&file);
-        ts << generatedAddExpectedFailureCode;
+    switch (mode) {
+    case TestConfig::Skip:
+        addTestExclusion(parts.at(0), parts.value(1));
+        break;
+
+    case TestConfig::ExpectFail:
+        addExpectedFailure(parts.at(0), parts.value(1),
+                           parts.value(2), parts.value(3));
+        break;
     }
-#endif
 }
 
 void tst_Suite::addExpectedFailure(const QString &testName, const QString &actual,