/**************************************************************************** ** ** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). ** All rights reserved. ** Contact: Nokia Corporation (qt-info@nokia.com) ** ** This file is part of the test suite of the Qt Toolkit. ** ** $QT_BEGIN_LICENSE:LGPL$ ** GNU Lesser General Public License Usage ** This file may be used under the terms of the GNU Lesser General Public ** License version 2.1 as published by the Free Software Foundation and ** appearing in the file LICENSE.LGPL included in the packaging of this ** file. Please review the following information to ensure the GNU Lesser ** General Public License version 2.1 requirements will be met: ** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. ** ** In addition, as a special exception, Nokia gives you certain additional ** rights. These rights are described in the Nokia Qt LGPL Exception ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. ** ** GNU General Public License Usage ** Alternatively, this file may be used under the terms of the GNU General ** Public License version 3.0 as published by the Free Software Foundation ** and appearing in the file LICENSE.GPL included in the packaging of this ** file. Please review the following information to ensure the GNU General ** Public License version 3.0 requirements will be met: ** http://www.gnu.org/copyleft/gpl.html. ** ** Other Usage ** Alternatively, this file may be used in accordance with the terms and ** conditions contained in a signed written agreement between you and Nokia. ** ** ** ** ** ** $QT_END_LICENSE$ ** ****************************************************************************/ #include #include #include #include #include #include #include #include #include #include #include #include static const char *ArthurDir = "../../arthur"; #include atWrapper::atWrapper() { // initTests(); } bool atWrapper::initTests(bool *haveBaseline) { qDebug() << "Running test on buildkey:" << QLibraryInfo::buildKey() << " qt version:" << qVersion(); qDebug( "Initializing tests..." ); if (!loadConfig( QHostInfo::localHostName().split( "." ).first() + ".ini" )) return false; //Reset the FTP environment where the results are stored *haveBaseline = setupFTP(); // Retrieve the latest test result baseline from the FTP server. downloadBaseline(); return true; } void atWrapper::downloadBaseline() { qDebug() << "Now downloading baseline..."; QFtp ftp; QObject::connect( &ftp, SIGNAL( listInfo( const QUrlInfo & ) ), this, SLOT( ftpMgetAddToList(const QUrlInfo & ) ) ); //Making sure that the needed local directories exist. QHashIterator j(enginesToTest); while ( j.hasNext() ) { j.next(); QDir dir( output ); if ( !dir.cd( j.key() + ".baseline" ) ) dir.mkdir( j.key() + ".baseline" ); } //FTP to the host specified in the config file, and retrieve the test result baseline. ftp.connectToHost( ftpHost ); ftp.login( ftpUser, ftpPass ); ftp.cd( ftpBaseDir ); QHashIterator i(enginesToTest); while ( i.hasNext() ) { i.next(); mgetDirList.clear(); mgetDirList << i.key() + ".baseline"; ftp.cd( i.key() + ".baseline" ); ftp.list(); ftp.cd( ".." ); while ( ftp.hasPendingCommands() ) QCoreApplication::instance()->processEvents(); ftpMgetDone( true ); } ftp.close(); ftp.close(); while ( ftp.hasPendingCommands() ) QCoreApplication::instance()->processEvents(); } void atWrapper::ftpMgetAddToList( const QUrlInfo &urlInfo ) { //Simply adding to the list of files to download. mgetDirList << urlInfo.name(); } void atWrapper::ftpMgetDone( bool error) { Q_UNUSED( error ); //Downloading the files listed in mgetDirList... QFtp ftp; ftp.connectToHost( ftpHost ); ftp.login( ftpUser, ftpPass ); QFile* file; if ( mgetDirList.size() > 1 ) for ( int i = 1; i < mgetDirList.size(); ++i ) { file = new QFile( QString( output ) + "/" + mgetDirList.at( 0 ) + "/" + mgetDirList.at( i ) ); if (file->open(QIODevice::WriteOnly)) { ftp.get( ftpBaseDir + "/" + mgetDirList.at( 0 ) + "/" + mgetDirList.at( i ), file ); ftp.list(); //Only there to fill up a slot in the pendingCommands queue. while ( ftp.hasPendingCommands() ) QCoreApplication::instance()->processEvents(); file->close(); } else { qDebug() << "Couldn't open file for writing: " << file->fileName(); } } while ( ftp.hasPendingCommands() ) QCoreApplication::instance()->processEvents(); } void atWrapper::uploadFailed( QString dir, QString filename, QByteArray filedata ) { //Upload a failed test case image to the FTP server. QFtp ftp; ftp.connectToHost( ftpHost ); ftp.login( ftpUser, ftpPass ); ftp.cd( ftpBaseDir ); ftp.cd( dir ); ftp.put( filedata, filename, QFtp::Binary ); ftp.close(); while ( ftp.hasPendingCommands() ) QCoreApplication::instance()->processEvents(); } // returns false if no baseline exists bool atWrapper::setupFTP() { qDebug( "Setting up FTP environment" ); QString dir = ""; ftpMkDir( ftpBaseDir ); ftpBaseDir += "/" + QLibraryInfo::buildKey(); ftpMkDir( ftpBaseDir ); ftpBaseDir += "/" + QString( qVersion() ); ftpMkDir( ftpBaseDir ); QHashIterator i(enginesToTest); QHashIterator j(enginesToTest); bool haveBaseline = true; //Creating the baseline directories for each engine while ( i.hasNext() ) { i.next(); //qDebug() << "Creating dir with key:" << i.key(); ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".failed" ); ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".diff" ); if (!ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".baseline" )) haveBaseline = false; } QFtp ftp; ftp.connectToHost( ftpHost ); ftp.login( ftpUser, ftpPass ); ftp.cd( ftpBaseDir ); //Deleting previous failed directory and all the files in it, then recreating it. while ( j.hasNext() ) { j.next(); rmDirList.clear(); rmDirList << ftpBaseDir + "/" + j.key() + ".failed" + "/"; ftpRmDir( j.key() + ".failed" ); ftp.rmdir( j.key() + ".failed" ); ftp.mkdir( j.key() + ".failed" ); ftp.list(); while ( ftp.hasPendingCommands() ) QCoreApplication::instance()->processEvents(); rmDirList.clear(); rmDirList << ftpBaseDir + "/" + j.key() + ".diff" + "/"; ftpRmDir( j.key() + ".diff" ); ftp.rmdir( j.key() + ".diff" ); ftp.mkdir( j.key() + ".diff" ); ftp.list(); while ( ftp.hasPendingCommands() ) QCoreApplication::instance()->processEvents(); } ftp.close(); while ( ftp.hasPendingCommands() ) QCoreApplication::instance()->processEvents(); return haveBaseline; } void atWrapper::ftpRmDir( QString dir ) { //Hack to remove a populated directory. (caveat: containing only files and empty dirs, not recursive!) qDebug() << "Now removing directory: " << dir; QFtp ftp; QObject::connect( &ftp, SIGNAL( listInfo( const QUrlInfo & ) ), this, SLOT( ftpRmDirAddToList(const QUrlInfo & ) ) ); QObject::connect( &ftp, SIGNAL( done( bool ) ), this, SLOT( ftpRmDirDone( bool ) ) ); ftp.connectToHost( ftpHost ); ftp.login( ftpUser, ftpPass ); ftp.list( ftpBaseDir + "/" + dir ); ftp.close(); ftp.close(); while ( ftp.hasPendingCommands() ) QCoreApplication::instance()->processEvents(); } void atWrapper::ftpRmDirDone( bool error ) { //Deleting each file in the directory listning, rmDirList. Q_UNUSED( error ); QFtp ftp; ftp.connectToHost( ftpHost ); ftp.login( ftpUser, ftpPass ); if ( rmDirList.size() > 1 ) for (int i = 1; i < rmDirList.size(); ++i) ftp.remove( rmDirList.at(0) + rmDirList.at( i ) ); ftp.close(); while ( ftp.hasPendingCommands() ) QCoreApplication::instance()->processEvents(); } // returns false if the directory already exists bool atWrapper::ftpMkDir( QString dir ) { //Simply used to avoid QFTP from bailing out and loosing a queue of commands. // IE: conveniance. QFtp ftp; QSignalSpy commandSpy(&ftp, SIGNAL(commandFinished(int, bool))); ftp.connectToHost( ftpHost ); ftp.login( ftpUser, ftpPass ); const int command = ftp.mkdir( dir ); ftp.close(); while ( ftp.hasPendingCommands() ) QCoreApplication::instance()->processEvents(); for (int i = 0; i < commandSpy.count(); ++i) if (commandSpy.at(i).at(0) == command) return commandSpy.at(i).at(1).toBool(); return false; } void atWrapper::ftpRmDirAddToList( const QUrlInfo &urlInfo ) { //Just adding the file to the list for deletion rmDirList << urlInfo.name(); } bool atWrapper::executeTests() { qDebug("Executing the tests..."); QHashIterator i(enginesToTest); DataGenerator generator; //Running datagenerator against all the frameworks specified in the config file. while ( i.hasNext() ) { i.next(); qDebug( "Now testing: " + i.key().toLatin1() ); char* params[13]; //./bin/datagenerator -framework data/framework.ini -engine OpenGL -suite 1.1 -output outtest QByteArray eng = i.key().toLatin1(); QByteArray fwk = framework.toLatin1(); QByteArray sut = suite.toLatin1(); QByteArray out = output.toLatin1(); QByteArray siz = size.toLatin1(); QByteArray fill = fillColor.toLatin1(); params[1] = "-framework"; params[2] = fwk.data(); params[3] = "-engine"; params[4] = eng.data(); params[5] = "-suite"; params[6] = sut.data(); params[7] = "-output"; params[8] = out.data(); params[9] = "-size"; params[10] = siz.data(); params[11] = "-fill"; params[12] = fill.data(); generator.run( 13, params ); } return true; } void atWrapper::createBaseline() { qDebug( "Now uploading a baseline of only the latest test values" ); QHashIterator i(enginesToTest); QDir dir( output ); QFtp ftp; ftp.connectToHost( ftpHost ); ftp.login( ftpUser, ftpPass ); ftp.cd( ftpBaseDir ); //Upload all the latest test results to the FTP server's baseline directory. while ( i.hasNext() ) { i.next(); dir.cd( i.key() ); ftp.cd( i.key() + ".baseline" ); dir.setFilter(QDir::Files | QDir::Hidden | QDir::NoSymLinks); dir.setNameFilters( QStringList() << "*.png" ); QFileInfoList list = dir.entryInfoList(); dir.cd( ".." ); for (int n = 0; n < list.size(); n++) { QFileInfo fileInfo = list.at( n ); QFile file( QString( output ) + "/" + i.key() + "/" + fileInfo.fileName() ); file.open( QIODevice::ReadOnly ); QByteArray fileData = file.readAll(); //qDebug() << "Sending up:" << fileInfo.fileName() << "with file size" << fileData.size(); file.close(); ftp.put( fileData, fileInfo.fileName(), QFtp::Binary ); } ftp.cd( ".." ); } ftp.close(); while ( ftp.hasPendingCommands() ) QCoreApplication::instance()->processEvents(); } bool atWrapper::compare() { qDebug( "Now comparing the results to the baseline" ); QHashIterator i(enginesToTest); while ( i.hasNext() ) { i.next(); compareDirs( output , i.key() ); } return true; } void atWrapper::compareDirs( QString basedir, QString target ) { QDir dir( basedir ); /* The following should be redundant now. if ( !dir.cd( target + ".failed" ) ) dir.mkdir( target + ".failed" ); else dir.cdUp(); */ if ( !dir.cd( target + ".diff" ) ) dir.mkdir( target + ".diff" ); else dir.cdUp(); //Perform comparisons between the two directories. dir.setFilter(QDir::Files | QDir::Hidden | QDir::NoSymLinks); dir.setNameFilters( QStringList() << "*.png" ); dir.cd( target + ".baseline" ); QFileInfoList list = dir.entryInfoList(); for (int i = 0; i < list.size(); ++i) { QFileInfo fileInfo = list.at(i); diff ( basedir, target, fileInfo.fileName() ); } } bool atWrapper::diff( QString basedir, QString dir, QString target ) { //Comparing the two specified files, and then uploading them to //the ftp server if they differ basedir += "/" + dir; QString one = basedir + ".baseline/" + target; QString two = basedir + "/" + target; QFile file( one ); file.open( QIODevice::ReadOnly ); QByteArray contentsOfOne = file.readAll(); file.close(); file.setFileName( two ); file.open( QIODevice::ReadOnly ); QByteArray contentsOfTwo = file.readAll(); file.close(); if ( contentsOfTwo.size() == 0 ) { qDebug() << "No test result found for baseline: " << one; file.setFileName( one ); file.open( QIODevice::ReadOnly ); file.copy( basedir + ".failed/" + target + "_missing" ); uploadFailed( dir + ".failed", target + "_missing", contentsOfTwo ); return false; } if ( ( memcmp( contentsOfOne, contentsOfTwo, contentsOfOne.size() ) ) == 0 ) return true; else { qDebug() << "Sorry, the result did not match: " << one; file.setFileName( two ); file.open( QIODevice::ReadOnly ); file.copy( basedir + ".failed/" + target ); file.close(); uploadFailed( dir + ".failed", target, contentsOfTwo ); uploadDiff( basedir, dir, target ); return false; } } void atWrapper::uploadDiff( QString basedir, QString dir, QString filename ) { qDebug() << basedir; QImage im1( basedir + ".baseline/" + filename ); QImage im2( basedir + "/" + filename ); QImage im3(im1.size(), QImage::Format_ARGB32); im1 = im1.convertToFormat(QImage::Format_ARGB32); im2 = im2.convertToFormat(QImage::Format_ARGB32); for ( int y=0; y