Added lancelot-based scenegraph rendering regression test

Change-Id: I6718d00ba96dc9c96dac82de4ded9228f6bfb990
Reviewed-by: Paul Olav Tvete <paul.tvete@digia.com>

1 files changed, 271 insertions, 0 deletions

diff --git a/tests/manual/scenegraph_lancelot/scenegraph/tst_scenegraph.cpp b/tests/manual/scenegraph_lancelot/scenegraph/tst_scenegraph.cpp
new file mode 100644
index 0000000000..fb26b80cc1
--- /dev/null
+++ b/tests/manual/scenegraph_lancelot/scenegraph/tst_scenegraph.cpp
@@ -0,0 +1,271 @@
+/****************************************************************************
+**
+** Copyright (C) 2012 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <qbaselinetest.h>
+
+#include <QtCore/QCoreApplication>
+#include <QtCore/QDirIterator>
+#include <QtCore/QDebug>
+#include <QtCore/QProcess>
+#include <QtGui/QImage>
+
+
+QString blockify(const QByteArray& s)
+{
+    const char* indent = "\n | ";
+    QByteArray block = s.trimmed();
+    block.replace('\n', indent);
+    block.prepend(indent);
+    block.append('\n');
+    return block;
+}
+
+
+class tst_Scenegraph : public QObject
+{
+    Q_OBJECT
+
+public:
+    tst_Scenegraph();
+
+private Q_SLOTS:
+    void initTestCase();
+    void cleanup();
+    void testNoTextRendering_data();
+    void testNoTextRendering();
+    void testRendering_data();
+    void testRendering();
+
+private:
+    void setupTestSuite(const QByteArray& filter = QByteArray());
+    void runTest(const QStringList& extraArgs = QStringList());
+    bool renderAndGrab(const QString& qmlFile, const QStringList& extraArgs, QImage *screenshot, QString *errMsg);
+    quint16 checksumFileOrDir(const QString &path);
+
+    int consecutiveErrors;   // Not test failures (image mismatches), but system failures (so no image at all)
+    bool aborted;            // This run given up because of too many system failures
+};
+
+
+tst_Scenegraph::tst_Scenegraph()
+    : consecutiveErrors(0), aborted(false)
+{
+}
+
+
+void tst_Scenegraph::initTestCase()
+{
+    QByteArray msg;
+    if (!QBaselineTest::connectToBaselineServer(&msg))
+        QSKIP(msg);
+}
+
+
+void tst_Scenegraph::cleanup()
+{
+    // Allow subsystems time to settle
+    if (!aborted)
+        QTest::qWait(200);
+}
+
+void tst_Scenegraph::testNoTextRendering_data()
+{
+    setupTestSuite("text/");
+    consecutiveErrors = 0;
+    aborted = false;
+}
+
+
+void tst_Scenegraph::testNoTextRendering()
+{
+    runTest(QStringList() << "-notext");
+}
+
+
+void tst_Scenegraph::testRendering_data()
+{
+    setupTestSuite();
+    consecutiveErrors = 0;
+    aborted = false;
+}
+
+
+void tst_Scenegraph::testRendering()
+{
+    runTest();
+}
+
+
+void tst_Scenegraph::setupTestSuite(const QByteArray& filter)
+{
+    QTest::addColumn<QString>("qmlFile");
+    int numItems = 0;
+
+    QString testSuiteDir = QLatin1String("data");
+    QString testSuiteLocation = QCoreApplication::applicationDirPath();
+    QString testSuitePath = testSuiteLocation + QDir::separator() + testSuiteDir;
+    QFileInfo fi(testSuitePath);
+    if (!fi.exists() || !fi.isDir() || !fi.isReadable())
+        QSKIP("Test suite data directory missing or unreadable: " + testSuitePath.toLatin1());
+
+    QStringList ignoreItems;
+    QFile ignoreFile(testSuitePath + "/Ignore");
+    if (ignoreFile.open(QIODevice::ReadOnly)) {
+        while (!ignoreFile.atEnd()) {
+            QByteArray line = ignoreFile.readLine().trimmed();
+            if (!line.isEmpty() && !line.startsWith('#'))
+                ignoreItems += line;
+        }
+    }
+
+    QStringList itemFiles;
+    QDirIterator it(testSuitePath, QDirIterator::Subdirectories);
+    while (it.hasNext()) {
+        QString fp = it.next();
+        if (fp.endsWith(".qml")) {
+            QString itemName = fp.mid(testSuitePath.length() + 1);
+            if (!ignoreItems.contains(itemName) && (filter.isEmpty() || !itemName.startsWith(filter)))
+                itemFiles.append(it.filePath());
+        }
+    }
+
+    qSort(itemFiles);
+    foreach (const QString &filePath, itemFiles) {
+        QByteArray itemName = filePath.mid(testSuitePath.length() + 1).toLatin1();
+        QBaselineTest::newRow(itemName, checksumFileOrDir(filePath)) << filePath;
+        numItems++;
+    }
+
+    if (!numItems)
+        QSKIP("No .qml test files found in " + testSuitePath.toLatin1());
+}
+
+
+void tst_Scenegraph::runTest(const QStringList& extraArgs)
+{
+    // qDebug() << "Rendering" << QTest::currentDataTag();
+
+    if (aborted)
+        QSKIP("System too unstable.");
+
+    QFETCH(QString, qmlFile);
+
+    QImage screenShot;
+    QString errorMessage;
+    if (renderAndGrab(qmlFile, extraArgs, &screenShot, &errorMessage)) {
+        consecutiveErrors = 0;
+    }
+    else {
+        if (++consecutiveErrors >= 3)
+            aborted = true;                   // Just give up if screen grabbing fails 3 times in a row
+        QFAIL(qPrintable("QuickView grabbing failed: " + errorMessage));
+    }
+
+    QBASELINE_TEST(screenShot);
+}
+
+
+bool tst_Scenegraph::renderAndGrab(const QString& qmlFile, const QStringList& extraArgs, QImage *screenshot, QString *errMsg)
+{
+    bool usePipe = true;  // Whether to transport the grabbed image using temp. file or pipe. TBD: cmdline option
+    QProcess grabber;
+    QString cmd = QCoreApplication::applicationDirPath() + "/qmlscenegrabber";
+    QStringList args = extraArgs;
+    QString tmpfile = usePipe ? QString("-") : QString("/tmp/qmlscenegrabber-%1-out.ppm").arg(QCoreApplication::applicationPid());
+    args << qmlFile << "-o" << tmpfile;
+    grabber.start(cmd, args, QIODevice::ReadOnly);
+    grabber.waitForFinished(17000);         //### hardcoded, must be larger than the scene timeout in qmlscenegrabber
+    if (grabber.state() != QProcess::NotRunning) {
+        grabber.terminate();
+        grabber.waitForFinished(3000);
+    }
+    QImage img;
+    bool res = usePipe ? img.load(&grabber, "ppm") : img.load(tmpfile);
+    if (!res || img.isNull()) {
+        if (errMsg) {
+            QString s("Failed to grab screen. qmlscenegrabber exitcode: %1. Process error: %2. Stderr:%3");
+            *errMsg = s.arg(grabber.exitCode()).arg(grabber.errorString()).arg(blockify(grabber.readAllStandardError()));
+        }
+        if (!usePipe)
+            QFile::remove(tmpfile);
+        return false;
+    }
+    if (screenshot)
+        *screenshot = img;
+    if (!usePipe)
+        QFile::remove(tmpfile);
+    return true;
+}
+
+
+quint16 tst_Scenegraph::checksumFileOrDir(const QString &path)
+{
+    QFileInfo fi(path);
+    if (!fi.exists() || !fi.isReadable())
+        return 0;
+    if (fi.isFile()) {
+        QFile f(path);
+        f.open(QIODevice::ReadOnly);
+        QByteArray contents = f.readAll();
+        return qChecksum(contents.constData(), contents.size());
+    }
+    if (fi.isDir()) {
+        static const QStringList nameFilters = QStringList() << "*.qml" << "*.cpp" << "*.png" << "*.jpg";
+        quint16 cs = 0;
+        foreach (QString item, QDir(fi.filePath()).entryList(nameFilters, QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot))
+            cs ^= checksumFileOrDir(path + "/" + item);
+        return cs;
+    }
+    return 0;
+}
+
+
+#define main _realmain
+QTEST_MAIN(tst_Scenegraph)
+#undef main
+
+int main(int argc, char *argv[])
+{
+    QBaselineTest::handleCmdLineArgs(&argc, &argv);
+    return _realmain(argc, argv);
+}
+
+#include "tst_scenegraph.moc"