From 64391c88c7e1421dd146b528c4395c68049c4adb Mon Sep 17 00:00:00 2001 From: Tim Jenssen Date: Tue, 8 Mar 2011 20:46:27 +0100 Subject: moved some directories and adjusted the README - also removed some unused projects under examples(updater, updaterplugin) - adjusted pro files to the new structure --- tests/README | 1 + tests/test-framework/README | 312 +++++++++++++++++++++ tests/test-framework/checker/run.py | 51 ++++ .../checker/scripts/generate-filelist.py | 41 +++ .../test-framework/checker/testrunner/__init__.py | 1 + tests/test-framework/checker/testrunner/files.py | 33 +++ tests/test-framework/checker/testrunner/logger.py | 0 .../test-framework/checker/testrunner/registry.py | 21 ++ .../checker/testrunner/testexception.py | 5 + .../checker/testrunner/testrunner.py | 90 ++++++ .../testcase-linux64/checker/testinstall.filelist | 5 + .../testcase-linux64/testcase-linux64.cfg | 5 + .../site/TestCases/testcase-linux64/testscript.qs | 67 +++++ .../testcase1/checker/testinstall.filelist | 5 + .../site/TestCases/testcase1/testcase1.cfg | 5 + .../site/TestCases/testcase1/testscript.qs | 67 +++++ .../site/VMConfigs/LinuxUbuntu9.1064Bit.cfg | 8 + .../site/VMConfigs/WindowsVista32Bit.cfg | 9 + .../site/VMConfigs/WindowsVista64Bit.cfg | 9 + .../site/VMConfigs/WindowsXp32Bit.cfg | 9 + .../site/VMConfigs/WindowsXp64Bit.cfg | 9 + tests/test-framework/site/host-config.cfg | 37 +++ tests/test-framework/site/listVMs.sh | 1 + tests/test-framework/tests/simpletest.py | 11 + tests/test-framework/tests/testfiles/test.filelist | 3 + tests/test-framework/vmware/cdashreporter.py | 156 +++++++++++ tests/test-framework/vmware/control.py | 220 +++++++++++++++ tests/test-framework/vmware/ftpsource.py | 117 ++++++++ tests/test-framework/vmware/guest.py | 160 +++++++++++ tests/test-framework/vmware/guestconfig.py | 11 + tests/test-framework/vmware/reporter.py | 96 +++++++ tests/test-framework/vmware/result.py | 148 ++++++++++ tests/test-framework/vmware/run-test.py | 111 ++++++++ tests/test-framework/vmware/run.py | 106 +++++++ tests/test-framework/vmware/source.py | 36 +++ tests/test-framework/vmware/testcase.py | 70 +++++ tests/test-framework/vmware/utils.py | 59 ++++ tests/test-framework/vmware/virtualmachine.py | 177 ++++++++++++ tests/test-framework/vmware/xmlutils.py | 13 + tests/testcases/result-example.xml | 37 +++ tests/testcases/testcase1/packagemanagement.qs | 35 +++ tests/testcases/testcase1/test-uninstall.qs | 30 ++ tests/testcases/testcase1/testcase1.cfg | 5 + tests/testcases/testcase1/testscript.qs | 69 +++++ tests/testreturn/main.cpp | 70 +++++ tests/testreturn/testreturn.pro | 10 + tests/testvm-configs/franks-host-config.cfg | 5 + tests/testvm-configs/franks-test-vm.cfg | 7 + tests/testvm-configs/mikes-host-config.cfg | 4 + tests/testvm-configs/mikes-test-vm.cfg | 7 + 50 files changed, 2564 insertions(+) create mode 100644 tests/README create mode 100644 tests/test-framework/README create mode 100644 tests/test-framework/checker/run.py create mode 100644 tests/test-framework/checker/scripts/generate-filelist.py create mode 100644 tests/test-framework/checker/testrunner/__init__.py create mode 100644 tests/test-framework/checker/testrunner/files.py create mode 100644 tests/test-framework/checker/testrunner/logger.py create mode 100644 tests/test-framework/checker/testrunner/registry.py create mode 100644 tests/test-framework/checker/testrunner/testexception.py create mode 100644 tests/test-framework/checker/testrunner/testrunner.py create mode 100644 tests/test-framework/site/TestCases/testcase-linux64/checker/testinstall.filelist create mode 100644 tests/test-framework/site/TestCases/testcase-linux64/testcase-linux64.cfg create mode 100644 tests/test-framework/site/TestCases/testcase-linux64/testscript.qs create mode 100644 tests/test-framework/site/TestCases/testcase1/checker/testinstall.filelist create mode 100644 tests/test-framework/site/TestCases/testcase1/testcase1.cfg create mode 100644 tests/test-framework/site/TestCases/testcase1/testscript.qs create mode 100644 tests/test-framework/site/VMConfigs/LinuxUbuntu9.1064Bit.cfg create mode 100644 tests/test-framework/site/VMConfigs/WindowsVista32Bit.cfg create mode 100644 tests/test-framework/site/VMConfigs/WindowsVista64Bit.cfg create mode 100644 tests/test-framework/site/VMConfigs/WindowsXp32Bit.cfg create mode 100644 tests/test-framework/site/VMConfigs/WindowsXp64Bit.cfg create mode 100644 tests/test-framework/site/host-config.cfg create mode 100644 tests/test-framework/site/listVMs.sh create mode 100644 tests/test-framework/tests/simpletest.py create mode 100644 tests/test-framework/tests/testfiles/test.filelist create mode 100644 tests/test-framework/vmware/cdashreporter.py create mode 100644 tests/test-framework/vmware/control.py create mode 100644 tests/test-framework/vmware/ftpsource.py create mode 100644 tests/test-framework/vmware/guest.py create mode 100644 tests/test-framework/vmware/guestconfig.py create mode 100644 tests/test-framework/vmware/reporter.py create mode 100644 tests/test-framework/vmware/result.py create mode 100644 tests/test-framework/vmware/run-test.py create mode 100644 tests/test-framework/vmware/run.py create mode 100644 tests/test-framework/vmware/source.py create mode 100644 tests/test-framework/vmware/testcase.py create mode 100644 tests/test-framework/vmware/utils.py create mode 100644 tests/test-framework/vmware/virtualmachine.py create mode 100644 tests/test-framework/vmware/xmlutils.py create mode 100644 tests/testcases/result-example.xml create mode 100644 tests/testcases/testcase1/packagemanagement.qs create mode 100644 tests/testcases/testcase1/test-uninstall.qs create mode 100644 tests/testcases/testcase1/testcase1.cfg create mode 100644 tests/testcases/testcase1/testscript.qs create mode 100644 tests/testreturn/main.cpp create mode 100644 tests/testreturn/testreturn.pro create mode 100644 tests/testvm-configs/franks-host-config.cfg create mode 100644 tests/testvm-configs/franks-test-vm.cfg create mode 100644 tests/testvm-configs/mikes-host-config.cfg create mode 100644 tests/testvm-configs/mikes-test-vm.cfg (limited to 'tests') diff --git a/tests/README b/tests/README new file mode 100644 index 000000000..06738ee93 --- /dev/null +++ b/tests/README @@ -0,0 +1 @@ +These are files for the unfinished test framework of the installer framework. \ No newline at end of file diff --git a/tests/test-framework/README b/tests/test-framework/README new file mode 100644 index 000000000..caa0c00a9 --- /dev/null +++ b/tests/test-framework/README @@ -0,0 +1,312 @@ += Testing Framework = + +== Overview == + +The "testing framework" was developed to continuously run automated, non-interactive tests for the Nokia NDK installers on a set of test systems. +It uses remote-controlled VMWare virtual machines (VMs) to execute the tests on. Both the actual controlling process and the post installation checks on the VMs are implemented in Python. +Additionally, a set of test cases is created and configured. +A test case consists of a) a QtScript script specifying the user input for non-interactive installation and b) post-installation checks to perform after installation. This way various user behaviour can be tested (e.g. different component selections) and whether installation succeeds and certain post-conditions are met (e.g. files end up in in specific locations). A test case can also contain additional steps to test updating and package management. +For each compatible VM/Test case combination, the VM is reset to a predefined state, the installer is run using the QtScript script and the post-installation checks are performed. +The test results are aggregated and submitted to a CDash dashboard. + +== Setup at Nokia == + + +The VMs run on a VMWare server installation on bersrv16723. The VMWare server web frontend requires a IE (32-bit) plugin to get the guest system's shell. +(Nokia internal: http://hegel.europe.nokia.com/binaries/vmware_client_plugin/vmware-vmrc-win32-x86.exe). +As far as I know there is no way atm to get it running on other platforms. + +Web frontend URL: http://bersrv16723:8222/ui (redirect to https://bersrv16723:8333/ui, but in some cases it is not working without this redirection) + +The testing framework itself is configured and runs on the controller VM. It it supposed to be accessed via ssh. + +Controller VM: 172.25.167.111 [get a fixed IP/hostname]. + +Attach/Start a screen session: + + screen -D -R + +Start the test runner: + + cd $HOME/test/installerfw/installer/test-framework + vmware/run.py -s 2010-05-17 site/host-config.cfg # insert your date here + +For more explanations on run.py see below. + +The CDash location presenting the build results is http://172.25.167.111/CDash (the controller VM). The project used is NDKInstallerTests (TODO: actually use that one). + +The current setup with working examples of configuration files is in $TODO:/home/kdab/test/ (TODO: commit to repo) + + +== run.py == + +The central python script executing test runs is vmware/run.py. + +Usage:run.py [options] host-config + +Options: + -s | --only-since YYYY-MM-DD when checking + + /path/to/vmware/run.py -s 2010-05-14 host-config.cfg + +run.py does the following: It collects the information about configured VMs and test cases from the host configuration file. +It then checks the configured installer sources for new installers, and sequentially executes the test for each new installer on all VM/test case combinations that apply to the installer (there can be specific tests cases for certain platforms only). For each installer x VM x test case run, the result is added to CDash. +Once all installers on the FTP server (or the ones matching the --only-since restriction) were tested, the process goes to sleep and checks every hour for new installers. + +run.py only executes all test runs sequentially, with a single test at a time. If this turns out to be too much of a bottleneck, multiple run.py instances in different working directories and with different +host-config files could be run in parallel (e.g. each one monitoring a different path on the FTP server). + +== Virtual Machine Setup == + +Any VMWare virtual machine can be used for testing, if some preparations are made: + + * Add a virtual CD/DVD drive (for VMWare Tools; Choose a random ISO file when asked) + * Install VMWare Tools (important: The test run might just hang and do nothing if the VM has no VMWare tools installed) + * Install Python 2.6 + * Create a snapshot of the VM in the state that should be used as initial state for each test run [TODO: relate to UI] + * Create a configuration file for the VM. The configuration file provides information to run.py about the VM, such as the user/password combination to use to login, the location of the python installation or the + path in the VM to use for temporary files, such as the downloaded installer and result output files. + (See Virtual Machine Configuration File Options for details) + * To enable the VM, add the path to the VM config file to the host config file (See Host Configuration File Options). + + +== Virtual Machine Configuration File Options == + +Example: + TODO + +=== Section: General === + +snapshot: identifier (Default: "base") + +The baseline snapshot to revert to when start a test run. +This option is only relevant VMWare Fusion and Workstation (as Server/ESX only support a single unnamed snapshot) + +username: string +password: string + +The username and password to authenticate with inside the VM. + +vmx: string + +The path to the VM's vmx file. Relative paths are understood relative to the configuration file. +For remote setups, the name as listed by the TODO tool must be used. +TODO: explain how to list snapshots. + +tempDir: guest path + +Path inside the VM to use for temporary data (installer, checker copy, test script...). +It must be writable by the user. + +os: windows|linux|mac + +The OS running inside the VM + +python: guest path (Default: "python") + +The path to the installed python executable inside the VM. + +== Test Cases == + +=== Configuration File === + +General/platforms: string list + +comma-separated list of platforms the testcase should be run on. +Valid platform names are "windows", "linux", "mac" + +General/targetDirectory: guest path + +The target directory to use for the post-installation checks (see below). Note that the actual installation directory is specified in the +installscript and must be kept in sync. + + +See the section on post-installation checks for details. + +General/maintenanceToolLocation + +The path to the maintenance tool (updater/package manager) after installation. Required for multi-step tests + +==== Steps ==== + +Each test case can consist of of n steps and must have at least one. +The first step is the actual installation and will execute the installer, the following steps will start the maintenance tool. +Step section must be consecutive and start at 0: Step0, Step1, Step2 etc. + +StepN/installscript: path + +The path to the QtScript used for non-interactive execution of the installer/maintenance tool. +relative paths are interpreted relative to the configuration file. + +StepN/checkerTestDir: string + +The path to a directory containing a set of tests to perform after a successful installer/maintenance tool run. +relative paths are interpreted relative to the configuration file. + +Example: + +[General] + +platforms=windows,linux +targetDirectory=c:\testinstall +maintenanceToolLocation=c:\testInstall\TestUninstaller.exe + +[Step0] +installscript=testscript.qs +checkerTestDir=checker + +=== Install scripts === + +see non-interactive installation section in installerbuilder/installerbuilder.dox +TODO: merge + +=== Post-installation checks === + +The framework can execute post-installation tests on the VM to check if the installation is in the expected state. +The python code executing those checks is called "checker". +Each testcase can contain a "checker test directory" to contain checks to be executed. +The framework copies both the checker code and the test case into the VM, executes the checks and reports the results. + +Currently, the following checks are supported: + +==== Files ==== + +It can be checked for files to exist in the installation and their file size and md5sum. +To add such a check, place a file with extension .filelist in the test directory. + +The format of a .filelist file is: + +filename; file size; md5sum + +file size and md5sum are optional and can be empty. + +Example: + +components.xml; 2050; 6813144fd09f7d39764702e5adb91679wrong +index.html; 46; fd40a94472ea1d13d93221c5ce62c321 +include\QtGui\qwidget.h; 108; 67dc776dd5aa66741dab6a2eeec4ac3c + +Relative paths are understood relative to the targetDirectory. +Such files can be generated using the script test-framework/checker/scripts/generate-filelist.py + +==== Windows Registry ==== + +Similar to .filelist, .registrylist files can be used to check for the existance and content of windows registry keys. + +The format is + +; ; + +HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion; SM_GamesName; Games + +==== Arbitrary python code ==== + +Not yet done but easy to implement: Run tests from python code from .py files in the test dir. + +== Host Configuration File Options == + +=== Section: General === + +vmrun: path (Default: "vmrun") + +vmrun specifies the location of the vmrun executable to use. vmrun is part of VMRun Workstation, Fusion, Server and ESX. If the +framework not on the same system as the VMWare server, you must sure that the executable installed supports the VMWare edition used +(At the time of writing, this is VMWare Server). + +checkerInstallation: path + +The path to the "checker" python files which will be copied to each VM and executed to perform postinstallation checks. + +Example: + +checkerInstallation=/home/kdab/test/installerfw/installer/test-framework/checker + +testcase{0..n}: path + +The configuration files of the test cases to use. +Relative paths are understood relative to the configuration file. + +vm{0..n}: path + +n configuration files of the virtual machines to use. +Relative paths are understood relative to the configuration file. + +gui: True|False (default: False TODO: ensure) + +Whether VMWare/vmrun should bring up . Only relevant when used with VMWare Workstation or Fusion. + +createErrorSnapshots True|False (default: False TODO: ensure) + +Specifies if the framework should create error snapshots if the installation or post-installation checks fail. +This is currently only supported for VMWare Workstation and Fusion (the server editions unfortunately only support a single snapshot per VM, +which is reserved for the baseline. + +=== Section: Host === + +This section is optional. It is used if the VMWare host is different from the local machine (VMWare Server or esx). + +type: string + +The VMWare edition to talk against, one of "server" and "esx" + +location: URL + +The URL of the server/esx installation to talk to + +Example: + +location=https://172.25.167.23:8333/sdk + +username: string + +The user used to login on the VMWare server + +password: string + +The user used to login on the VMWare server + +=== Section: CDash === + +The framework is able to generate test reports in CDash format. If this section does not exist, the results are just printed to stdout. + +host: hostname + +The host of the CDash installation + +location: + +The path to the CDash installation on host + + +project + +The CDash project to report the test results to. + +Example: + +[Host] + +host=localhost +location=/CDash +project=NokiaSDKInstallerTest + +=== Section: Source{0..n} === + + +Each source provides installers to be tested, for a specific platform. +Currently, FTP is assumed and the only supported protocol. +The + +Example: + +[Source0] +host=hegel +path=/projects/ndk/installers/windows +platform=windows + +[Source1] +host=hegel +path=/projects/ndk/installers/linux +platform=linux + + diff --git a/tests/test-framework/checker/run.py b/tests/test-framework/checker/run.py new file mode 100644 index 000000000..137fafb94 --- /dev/null +++ b/tests/test-framework/checker/run.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +import optparse, sys +from testrunner import testrunner + +from xml.sax.saxutils import XMLGenerator +from xml.sax.xmlreader import AttributesNSImpl + +class Writer: + def __init__( self, out ): + self._out = out + self.gen = XMLGenerator( out, 'utf-8' ) + self.gen.startDocument() + self.gen.startElement( 'results', {} ) + + def addResult( self, name, status, errstr ): + self.gen.startElement('result', { 'name':name, 'status':status } ) + self.gen.characters( errstr ) + self.gen.endElement('result' ) + + def addFailed( self, name, errstr ): + self.addResult( name, "failed", errstr ) + + def addPassed( self, status, errstr ): + self.addResult( name, "passed", errstr ) + + def finalize( self ): + self.gen.endElement( 'results' ) + self.gen.endDocument + self._out.write( '\n' ) + +optionParser = optparse.OptionParser(usage="%prog [options] testcaseDir", version="%prog 0.1") +optionParser.add_option("-p", "--omit-prefix", dest="prefix", help="for file checks, prefix relative paths with this prefix", metavar="PREFIX" ) +optionParser.add_option("-o", "--output", dest="output", help="write results to file (instead of stdout)", metavar="OUTPUT" ) +(options, args) = optionParser.parse_args() + +try: + testdir = args[0] +except IndexError: + optionParser.print_usage( sys.stderr ) + sys.exit( 1 ) + +out = sys.stdout +if options.output != None: + out = file( options.output, 'wb' ) + +writer = Writer( out ) +try: + runner = testrunner.TestRunner( testdir, options.prefix, writer ) + runner.run() +finally: + writer.finalize() \ No newline at end of file diff --git a/tests/test-framework/checker/scripts/generate-filelist.py b/tests/test-framework/checker/scripts/generate-filelist.py new file mode 100644 index 000000000..ba4929304 --- /dev/null +++ b/tests/test-framework/checker/scripts/generate-filelist.py @@ -0,0 +1,41 @@ +#!/usr/bin/python +import optparse, os, sys +from testrunner import files, testrunner + +class GenerateException( Exception ): + def __init__( self, value ): + self.value = value + def __str__( self ): + return repr( self.value ) + +def relpath( path, prefix ): + if prefix != None: + return os.path.relpath( path, prefix ) + else: + return path; + +out = sys.stdout + +def walker( prefix, current_dir, children ): + for c in children: + child = current_dir + os.sep + c + if os.path.isdir( child ): + continue + fileObj = file( child, 'rb' ) + md5 = files.md5sum( fileObj ) + out.write( "{0}; {1}; {2}\n".format( relpath( child, prefix ), os.path.getsize( child ), md5 ) ) + +optionParser = optparse.OptionParser(usage="%prog [options] directory", version="%prog 0.1") +optionParser.add_option("-p", "--omit-prefix", dest="prefix", help="make entries relative to this prefix", metavar="PREFIX" ) +optionParser.add_option("-o", "--output", dest="output", help="save file list to file (instead of stdout)", metavar="OUTPUT" ) +(options, args) = optionParser.parse_args() + +try: + directory = args[0] +except IndexError: + raise GenerateException( "No directory given.") + +if options.output != None: + out = file( options.output, 'wb' ) + +os.path.walk( directory, walker, options.prefix ) diff --git a/tests/test-framework/checker/testrunner/__init__.py b/tests/test-framework/checker/testrunner/__init__.py new file mode 100644 index 000000000..210f31223 --- /dev/null +++ b/tests/test-framework/checker/testrunner/__init__.py @@ -0,0 +1 @@ +# init.py diff --git a/tests/test-framework/checker/testrunner/files.py b/tests/test-framework/checker/testrunner/files.py new file mode 100644 index 000000000..629bedd05 --- /dev/null +++ b/tests/test-framework/checker/testrunner/files.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +from testexception import TestException + +import fnmatch, hashlib, os + +def md5sum( fileObj ): + md5 = hashlib.md5() + while True: + chunk = fileObj.read( 4096 ) + if not chunk: + break + md5.update( chunk ) + return md5.hexdigest() + +def locateFiles( rootPath, pattern ): + for path, dirs, files in os.walk( os.path.abspath( rootPath ) ): + for filename in fnmatch.filter( files, pattern ): + yield os.path.join( path, filename ) + +def checkFileImpl( path, expectedSize=-1, expectedMd5=None ): + #TODO: normalize path/convert to platform + if not os.path.exists( path ): + raise TestException( '{0}: file does not exist'.format( path ) ) + size = os.path.getsize( path ) + if expectedSize >= 0 and size != expectedSize: + raise TestException( '{0}: unexpected size. Actual: {1} Expected: {2}'.format( path, size, expectedSize ) ) + if ( expectedMd5 != None ): + fileObj = file( path, 'rb' ) + md5 = md5sum( fileObj ) + fileObj.close() + if md5 != expectedMd5: + raise TestException( '{0}: md5sum mismatch. Actual: {1} Expected: {2}'.format( path, md5, expectedMd5 ) ) + diff --git a/tests/test-framework/checker/testrunner/logger.py b/tests/test-framework/checker/testrunner/logger.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test-framework/checker/testrunner/registry.py b/tests/test-framework/checker/testrunner/registry.py new file mode 100644 index 000000000..854a029f3 --- /dev/null +++ b/tests/test-framework/checker/testrunner/registry.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from testexception import TestException +import _winreg + +_registry = dict() +_registry["HKEY_CLASSES_ROOT"] = _winreg.HKEY_CLASSES_ROOT +_registry["HKEY_CURRENT_USER"] = _winreg.HKEY_CURRENT_USER +_registry["HKEY_LOCAL_MACHINE"] = _winreg.HKEY_LOCAL_MACHINE +_registry["HKEY_USERS"] = _winreg.HKEY_USERS +_registry["HKEY_CURRENT_CONFIG"] = _winreg.HKEY_CURRENT_CONFIG + +def splitKey( key ): + key, seperator, subKey = key.partition( '\\' ) + return _registry[key], subKey + +def checkKey( key, value, expectedData ): + baseKey, subKey = splitKey( key ) + keyHandle = _winreg.OpenKey( baseKey, subKey ) + data, _ = _winreg.QueryValueEx( keyHandle, value ) + if data != expectedData: + raise TestException( '{0}: unexpected registry data. Actual: {1} Expected: {2}'.format( key, data, expectedData ) ) diff --git a/tests/test-framework/checker/testrunner/testexception.py b/tests/test-framework/checker/testrunner/testexception.py new file mode 100644 index 000000000..58254a599 --- /dev/null +++ b/tests/test-framework/checker/testrunner/testexception.py @@ -0,0 +1,5 @@ +class TestException( Exception ): + def __init__( self, value ): + self.value = value + def __str__( self ): + return repr( self.value ) diff --git a/tests/test-framework/checker/testrunner/testrunner.py b/tests/test-framework/checker/testrunner/testrunner.py new file mode 100644 index 000000000..fcef00b02 --- /dev/null +++ b/tests/test-framework/checker/testrunner/testrunner.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +import files, os, string, platform +from testexception import TestException + +if ( platform.system() == "Windows" ): + import registry + +def makeAbsolutePath( path, relativeTo ): + if os.path.isabs( path ) or relativeTo == None: + return path + else: + return relativeTo + os.sep + path + + +class TestRunner: + def __init__( self, testDir, basedir, result ): + self._testDir = testDir + self._basedir = basedir + self._result = result + + def checkFile( self, name, size=-1, expectedMd5=None ): + try: + files.checkFileImpl( name, size, expectedMd5 ) + except TestException as e: + self._result.addFailed( name, e.value ) + + def checkFileList( self, path ): + lineNum = 0 + haveError = False + with open( path, 'r' ) as f: + while True: + line = f.readline() + lineNum += 1 + if not line: + break + line = string.strip( line ) + if len( line ) == 0: + continue + segments = string.split( line, ';' ) + if len( segments ) == 3: + fp = makeAbsolutePath( segments[0], self._basedir ) + try: + fs = int( segments[1] ) + except ValueError: + fs = -1 #TODO handle error + femd5 = segments[2] + femd5 = string.strip( femd5 ) + self.checkFile( fp, fs, femd5 ) + else: + self._result.addFailed( path + '_' + str( lineNum ), "Could not parse file list entry: " + line ) + haveError = True + if not haveError: + self._result.addPassed( path, "" ) + + def checkRegistryList( self, path ): + haveError = False + lineNum = 0 + with open( path, 'r' ) as f: + while True: + lineNum += 1 + line = f.readline() + if not line: + break + line = string.strip( line ) + if len( line ) == 0: + continue + segments = string.split( line, ';' ) + if len( segments ) == 3: + key = segments[0].strip() + value = segments[1].strip() + expectedData = segments[2].strip() + registry.checkKey( key, value, expectedData ) + else: + self._result.addFailed( path + '_' + str( lineNum ), "Could not parse registry list entry: " + line ) + haveError = True + if not haveError: + self._result.addPassed( path, "" ) + + def run( self ): + fileLists = files.locateFiles( self._testDir, "*.filelist" ) + for i in fileLists: + self.checkFileList( i ) + + if ( platform.system() == "Windows" ): + registryLists = files.locateFiles( self._testDir, "*.registrylist" ) + for i in registryLists: + self.checkRegistryList( i ) + + # run all .py's in testdir + # execute all filelists diff --git a/tests/test-framework/site/TestCases/testcase-linux64/checker/testinstall.filelist b/tests/test-framework/site/TestCases/testcase-linux64/checker/testinstall.filelist new file mode 100644 index 000000000..30aecdabc --- /dev/null +++ b/tests/test-framework/site/TestCases/testcase-linux64/checker/testinstall.filelist @@ -0,0 +1,5 @@ +components.xml; 2050; 6813144fd09f7d39764702e5adb91679wrong +index.html; 46; fd40a94472ea1d13d93221c5ce62c321 +uninstall.exe; 13598393; 44945e7d3507d294b5e9e096ac3269b1 +include\QtCore\qobject.h; 108; 9e50d789f32d1651e16b6ae55699eb71 +include\QtGui\qwidget.h; 108; 67dc776dd5aa66741dab6a2eeec4ac3c diff --git a/tests/test-framework/site/TestCases/testcase-linux64/testcase-linux64.cfg b/tests/test-framework/site/TestCases/testcase-linux64/testcase-linux64.cfg new file mode 100644 index 000000000..ffbc24556 --- /dev/null +++ b/tests/test-framework/site/TestCases/testcase-linux64/testcase-linux64.cfg @@ -0,0 +1,5 @@ +[General] +installscript=testscript.qs +platforms=linux64 +targetDirectory=/home/kdab/testinstall +checkerTestDir=checker diff --git a/tests/test-framework/site/TestCases/testcase-linux64/testscript.qs b/tests/test-framework/site/TestCases/testcase-linux64/testscript.qs new file mode 100644 index 000000000..3cc4bd4d6 --- /dev/null +++ b/tests/test-framework/site/TestCases/testcase-linux64/testscript.qs @@ -0,0 +1,67 @@ +function Controller() +{ + installer.autoRejectMessageBoxes + installer.setMessageBoxAutomaticAnswer( "overwriteTargetDirectory", QMessageBox.Yes ) +} + + +Controller.prototype.IntroductionPageCallback = function() +{ + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.LicenseAgreementPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "LicenseAgreementPage" ) + page.acceptLicenseRB.setChecked( true ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.TargetDirectoryPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "TargetDirectoryPage" ) + page.targetDirectoryLE.setText( "/home/kdab/testinstall" ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.ComponentSelectionPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "ComponentSelectionPage" ) + page.deselectComponent( "com.nokia.ndk.tools.maemo.usbdriver" ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.DynamicQtGuiPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "DynamicQtGuiPage" ) + page.checkBoxLib.setChecked( false ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.DynamicErrorPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "DynamicErrorPage" ) + page.checkBoxMakeSure.setChecked( true ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.ReadyForInstallationPageCallback = function() +{ + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.StartMenuDirectoryPageCallback = function() +{ + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.PerformInstallationPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "PerformInstallationPage" ) + page.details.button.click +} + +Controller.prototype.FinishedPageCallback = function() +{ + gui.clickButton( buttons.FinishButton ) +} diff --git a/tests/test-framework/site/TestCases/testcase1/checker/testinstall.filelist b/tests/test-framework/site/TestCases/testcase1/checker/testinstall.filelist new file mode 100644 index 000000000..30aecdabc --- /dev/null +++ b/tests/test-framework/site/TestCases/testcase1/checker/testinstall.filelist @@ -0,0 +1,5 @@ +components.xml; 2050; 6813144fd09f7d39764702e5adb91679wrong +index.html; 46; fd40a94472ea1d13d93221c5ce62c321 +uninstall.exe; 13598393; 44945e7d3507d294b5e9e096ac3269b1 +include\QtCore\qobject.h; 108; 9e50d789f32d1651e16b6ae55699eb71 +include\QtGui\qwidget.h; 108; 67dc776dd5aa66741dab6a2eeec4ac3c diff --git a/tests/test-framework/site/TestCases/testcase1/testcase1.cfg b/tests/test-framework/site/TestCases/testcase1/testcase1.cfg new file mode 100644 index 000000000..178613640 --- /dev/null +++ b/tests/test-framework/site/TestCases/testcase1/testcase1.cfg @@ -0,0 +1,5 @@ +[General] +installscript=testscript.qs +platforms=windows,linux +targetDirectory=c:\testinstall +checkerTestDir=checker diff --git a/tests/test-framework/site/TestCases/testcase1/testscript.qs b/tests/test-framework/site/TestCases/testcase1/testscript.qs new file mode 100644 index 000000000..59bf25e3c --- /dev/null +++ b/tests/test-framework/site/TestCases/testcase1/testscript.qs @@ -0,0 +1,67 @@ +function Controller() +{ + installer.autoRejectMessageBoxes + installer.setMessageBoxAutomaticAnswer( "overwriteTargetDirectory", QMessageBox.Yes ) +} + + +Controller.prototype.IntroductionPageCallback = function() +{ + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.LicenseAgreementPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "LicenseAgreementPage" ) + page.acceptLicenseRB.setChecked( true ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.TargetDirectoryPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "TargetDirectoryPage" ) + page.targetDirectoryLE.setText( "c:\\testinstall" ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.ComponentSelectionPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "ComponentSelectionPage" ) + page.deselectComponent( "com.nokia.ndk.tools.maemo.usbdriver" ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.DynamicQtGuiPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "DynamicQtGuiPage" ) + page.checkBoxLib.setChecked( false ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.DynamicErrorPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "DynamicErrorPage" ) + page.checkBoxMakeSure.setChecked( true ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.ReadyForInstallationPageCallback = function() +{ + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.StartMenuDirectoryPageCallback = function() +{ + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.PerformInstallationPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "PerformInstallationPage" ) + page.details.button.click +} + +Controller.prototype.FinishedPageCallback = function() +{ + gui.clickButton( buttons.FinishButton ) +} diff --git a/tests/test-framework/site/VMConfigs/LinuxUbuntu9.1064Bit.cfg b/tests/test-framework/site/VMConfigs/LinuxUbuntu9.1064Bit.cfg new file mode 100644 index 000000000..a5c19e084 --- /dev/null +++ b/tests/test-framework/site/VMConfigs/LinuxUbuntu9.1064Bit.cfg @@ -0,0 +1,8 @@ +[General] +snapshot=base +username=kdab +password=kdab +vmx=[standard] Linux Ubuntu 9.10 64 Bit/Linux Ubuntu 9.10 64 Bit.vmx +tempDir=/home/kdab +os=linux64 +python=/usr/bin/python diff --git a/tests/test-framework/site/VMConfigs/WindowsVista32Bit.cfg b/tests/test-framework/site/VMConfigs/WindowsVista32Bit.cfg new file mode 100644 index 000000000..40a49d2f1 --- /dev/null +++ b/tests/test-framework/site/VMConfigs/WindowsVista32Bit.cfg @@ -0,0 +1,9 @@ +[General] +snapshot=base +username=kdab +password=kdab +vmx=[standard] Windows Vista 32 Bit/Windows Vista 32 Bit.vmx +tempDir=c:\Users\kdab\Desktop +os=windows +python=c:\Python26\python.exe + diff --git a/tests/test-framework/site/VMConfigs/WindowsVista64Bit.cfg b/tests/test-framework/site/VMConfigs/WindowsVista64Bit.cfg new file mode 100644 index 000000000..c46d8fba5 --- /dev/null +++ b/tests/test-framework/site/VMConfigs/WindowsVista64Bit.cfg @@ -0,0 +1,9 @@ +[General] +snapshot=base +username=kdab +password=kdab +vmx=[standard] Windows XP 32 Bit/Windows XP 32 Bit.vmx +tempDir=c:\Dokumente und Einstellungen\kdab\Desktop +os=windows +python=c:\Python26\python.exe + diff --git a/tests/test-framework/site/VMConfigs/WindowsXp32Bit.cfg b/tests/test-framework/site/VMConfigs/WindowsXp32Bit.cfg new file mode 100644 index 000000000..c46d8fba5 --- /dev/null +++ b/tests/test-framework/site/VMConfigs/WindowsXp32Bit.cfg @@ -0,0 +1,9 @@ +[General] +snapshot=base +username=kdab +password=kdab +vmx=[standard] Windows XP 32 Bit/Windows XP 32 Bit.vmx +tempDir=c:\Dokumente und Einstellungen\kdab\Desktop +os=windows +python=c:\Python26\python.exe + diff --git a/tests/test-framework/site/VMConfigs/WindowsXp64Bit.cfg b/tests/test-framework/site/VMConfigs/WindowsXp64Bit.cfg new file mode 100644 index 000000000..effbbd70d --- /dev/null +++ b/tests/test-framework/site/VMConfigs/WindowsXp64Bit.cfg @@ -0,0 +1,9 @@ +[General] +snapshot=base +username=kdab +password=kdab +vmx=[standard] Windows XP 64 Bit/Windows XP 64 Bit.vmx +tempDir=c:\Documents and Settings\kdab\Desktop +os=windows +python=c:\Python26\python.exe + diff --git a/tests/test-framework/site/host-config.cfg b/tests/test-framework/site/host-config.cfg new file mode 100644 index 000000000..f500e6fc9 --- /dev/null +++ b/tests/test-framework/site/host-config.cfg @@ -0,0 +1,37 @@ +[General] +vmrun=vmrun +checkerInstallation=/home/kdab/test/installerfw/installer/test-framework/checker +testcase0=TestCases/testcase1/testcase1.cfg +testcase1=TestCases/testcase-linux64/testcase-linux64.cfg +vm0=VMConfigs/WindowsXp32Bit.cfg +vm1=VMConfigs/WindowsXp64Bit.cfg +vm2=VMConfigs/WindowsVista32Bit.cfg +vm3=VMConfigs/LinuxUbuntu9.1064Bit.cfg +gui=False +createErrorSnapshots=False + +[Host] +type=server +location=https://172.25.167.23:8333/sdk +username=kdab +password=kdab + +[CDash] +host=localhost +location=/CDash +project=test1 + +[Source0] +host=hegel +path=/projects/ndk/installers/windows +platform=windows + +[Source1] +host=hegel +path=/projects/ndk/installers/linux/x32 +platform=linux32 + +[Source2] +host=hegel +path=/projects/ndk/installers/linux/x64 +platform=linux64 diff --git a/tests/test-framework/site/listVMs.sh b/tests/test-framework/site/listVMs.sh new file mode 100644 index 000000000..e9d5342d4 --- /dev/null +++ b/tests/test-framework/site/listVMs.sh @@ -0,0 +1 @@ +vmrun -u kdab -p kdab -T server -h https://172.25.167.23:8333/sdk listRegisteredVM diff --git a/tests/test-framework/tests/simpletest.py b/tests/test-framework/tests/simpletest.py new file mode 100644 index 000000000..5a320191a --- /dev/null +++ b/tests/test-framework/tests/simpletest.py @@ -0,0 +1,11 @@ +import sys +from testrunner import testrunner + +class Result: + def addError( self, errstr ): + print errstr + + +result = Result() +runner = testrunner.TestRunner( sys.argv[1], result ) +runner.run() diff --git a/tests/test-framework/tests/testfiles/test.filelist b/tests/test-framework/tests/testfiles/test.filelist new file mode 100644 index 000000000..fdeb8ee2f --- /dev/null +++ b/tests/test-framework/tests/testfiles/test.filelist @@ -0,0 +1,3 @@ +/home/frank/.ssh/id_dsa.pub; 1113; 1fddf250c364370b7936c1b7256c7eda +/home/frank/.ssh/id_dsa.pub; 1113; 1fddf250c364370b7936c1b7256c7edb + diff --git a/tests/test-framework/vmware/cdashreporter.py b/tests/test-framework/vmware/cdashreporter.py new file mode 100644 index 000000000..5acaa783e --- /dev/null +++ b/tests/test-framework/vmware/cdashreporter.py @@ -0,0 +1,156 @@ +# -*- coding: utf-8 -*- +import datetime, socket, os, sys, traceback, time, tempfile, httplib, urllib +from xml.sax.saxutils import XMLGenerator +from xml.sax.xmlreader import AttributesNSImpl +import result, utils +from result import exitStatusAsString +from xmlutils import startElement, endElement, writeElement + +checkerResultAsString = { result.CheckerResult.Passed:u"passed", result.CheckerResult.Failed:u"failed", result.CheckerResult.NotRun:u"notRun" } + +def formatDate( date ): + return date.strftime( '%Y%m%d-%H%M' ) + +def writeExecutionResult( gen, name, r ): + if r == None or r.hasError(): + stat = "failed" + else: + stat = "passed" + + startElement( gen, None, "Test", { u"Status":stat } ) + writeElement( gen, None, "Name", name ) + writeElement( gen, None, "FullName", name ) + startElement( gen, None, "NamedMeasurement", { u"type":u"text/string", u"name":u"Exit Code"} ) + if r: + msg = "(Unexpected)" + ec = r.exitCode + if ec == 0: + msg = "(Success)" + elif ec == 1: + msg = "(Failed)" + elif ec == 2: + msg = "(Canceled)" + + writeElement( gen, None, "Value", "Exit status: {0}; Installer exit code: {1} {2}".format( exitStatusAsString( r.exitStatus ), str( r.exitCode ), msg ) ) + else: + writeElement( gen, None, "Value", "Could not determine installation result." ) + + endElement( gen, None, "NamedMeasurement" ) + + + if r: + startElement( gen, None, "NamedMeasurement", { u"type":u"numeric/double", u"name":u"Execution Time"} ) + writeElement( gen, None, "Value", str( r.executionTime ) ) + endElement( gen, None, "NamedMeasurement" ) + + + endElement( gen, None, "Test" ) + +def writeInternalError( gen, errorstr, num ): + startElement( gen, None, "Test", { u"Status":u"failed"} ) + name = "InternalError{0}".format( num ) + writeElement( gen, None, "Name", name ) + writeElement( gen, None, "FullName", name ) + startElement( gen, None, "NamedMeasurement", { u"type":u"text/string", u"name":u"Completion Status"} ) + writeElement( gen, None, "Value", errorstr ) + endElement( gen, None, "NamedMeasurement" ) + endElement( gen, None, "Test" ) + +def writeTest( gen, test ): + startElement( gen, None, "Test", { u"Status":checkerResultAsString[test.result]} ) + writeElement( gen, None, "Name", test.name ) + writeElement( gen, None, "FullName", test.name ) + startElement( gen, None, "NamedMeasurement", { u"type":u"text/string", u"name":u"Completion Status"} ) + writeElement( gen, None, "Value", test.errorString ) + endElement( gen, None, "NamedMeasurement" ) + endElement( gen, None, "Test" ) + +class CDashReporter: + def __init__( self, host, location, project ): + self._host = host + self._location = location + self._project = project + + def reportException( self ): + sys.stderr.write( traceback.format_exc( 15 ) + '\n' ) + #TODO + + def reportResult( self, result ): + self._buildStamp = '{0}-Nightly'.format( datetime.datetime.now().strftime( '%Y%m%d-%H%M' ), utils.randomString( 4 ) ) + try: + #generate XML + self.genXml( self.genTestXml, result ) + #generate Tests.xml + #submitFile('Tests.xml') + except: + raise + + def genTestXml( self, gen, result ): + name = "{0} on {1}".format( result._installer.sourceFilename, result._vm.name() ) + startElement( gen, None, "Site", { u"BuildStamp":self._buildStamp, u"Name":name, u"Generator":u"VMTester 0.1" } ) + startElement( gen, None, "Testing" ) + writeElement( gen, None, "StartDateTime", formatDate( result._testStart ) ) + startElement( gen, None, "TestList" ) + internal = result._internalErrors + for i in range( 0, len( internal ) - 1 ): + writeElement( gen, None, "Test", "InternalError{0}".format( i ) ) + + for stepNum in range( len( result._stepResults ) ): + step = result._stepResults[stepNum] + writeElement( gen, None, "Test", "installer-run{0}".format( stepNum ) ) + for i in step.checkerResults: + writeElement( gen, None, "Test", i.name ) + endElement( gen, None, "TestList" ) + + for i in range( 0, len( internal ) - 1 ): + writeInternalError( gen, internal[i], i ) + for stepNum in range( len( result._stepResults ) ): + step = result._stepResults[stepNum] + writeExecutionResult( gen, "installer-run{0}".format( stepNum ), step.executionResult ) + for i in step.checkerResults: + writeTest( gen, i ) + endElement( gen, None, "Testing" ) + endElement( gen, None, "Site" ) + + def genXml( self, fillFunc, result ): + f = None + try: + f = tempfile.NamedTemporaryFile( delete=False ) + gen = XMLGenerator( f, 'utf-8' ) + gen.startDocument() + fillFunc( gen, result ) + gen.endDocument() + f.close() + self.submitFile( f.name ) + finally: + if f: + os.unlink( f.name ) + + def submitFile( self, path ): + f = file( path ) + params = urllib.urlencode( {'project': self._project} ) + conn = None + try: + try: + conn = httplib.HTTPConnection( self._host ) + conn.request( "POST", "{0}/submit.php?project={1}".format( self._location, self._project ), f ) + response = conn.getresponse() + print response.status, response.reason + #TODO check result + finally: + if conn: + conn.close() + except: + #TODO: if submitting to cdash fails, try to notify the admin (mail?) + raise + +if __name__ == "__main__": + r = result.Result() + r.testStarted() + r.addCheckerResult( result.CheckerResult( "test1", result.CheckerResult.Passed, "" ) ) + r.addCheckerResult( result.CheckerResult( "test2", result.CheckerResult.Failed, "Something went wrong, dude!" ) ) + time.sleep( 1 ) + r.testFinished() + cr = CDashReporter( "http://localhost", "/CDash", "test1" ) + cr.reportResult( r ) + diff --git a/tests/test-framework/vmware/control.py b/tests/test-framework/vmware/control.py new file mode 100644 index 000000000..b24c33ea9 --- /dev/null +++ b/tests/test-framework/vmware/control.py @@ -0,0 +1,220 @@ +# -*- coding: utf-8 -*- +import ConfigParser, datetime, os, string, sys, time, platform +import testcase, utils, result, virtualmachine +from virtualmachine import VMException +from xml.sax import make_parser +from xml.sax.handler import ContentHandler + +class ControlException( Exception ): + def __init__( self, value ): + self.value = value + def __str__( self ): + return repr( self.value ) + +class Handler( ContentHandler ): + def __init__( self, res ): + self._res = res + self._inResult = False + self._buf = "" + + def startElement( self, name, attrs ): + self._inResult = False + if name == "result": + self._inResult = True + self._name = attrs['name'] + self._status = attrs['status'] + + def endElement( self, name ): + if name == 'result': + trimmed = string.strip( self._buf ) + if self._status == "passed": + stat = result.CheckerResult.Passed + else: + stat = result.CheckerResult.Failed + self._res.addCheckerResult( result.CheckerResult( self._name, stat, trimmed ) ) + self._inResult = False + + self._buf = "" + + def characters( self, ch ): + if self._inResult: + self._buf += ch + +class Control: + def __init__( self, vmrun, checkerDir, source, reporter ): + self._vmrun = vmrun + self._checkerDir = checkerDir + self._vms = [] + self._testcases = [] + self._source = source + self._reporter = reporter + self._guiEnabled = True + self._createErrorSnapshots = False + self._hostType = "" + self._hostLocation = "" + self._hostUsername = "" + self._hostPassword = "" + + def setGuiEnabled( self, usegui ): + self._guiEnabled = usegui + for i in self._vms: + i.setGuiEnabled( usegui ) + + def setCreateErrorSnapshots( self, createSnapshots ): + self._createErrorSnapshots = createSnapshots + + def setRemoteHost( self, type, loc, user, pw ): + self._hostType = type + self._hostLocation = loc + self._hostUsername = user + self._hostPassword = pw + for vm in self._vms: + if not vm.isRemote(): + vm.setRemoteHost( self._hostType, self._hostLocation, self._hostUsername, self._hostPassword ) + + def addVM( self, cfgpath ): + config = ConfigParser.SafeConfigParser() + config.read( cfgpath ) + vm = virtualmachine.fromVMRunAndPath( self._vmrun, cfgpath ) + vm.setGuiEnabled( self._guiEnabled ) + if len( self._hostType ) > 0 and not vm.isRemote(): + vm.setRemoteHost( self._hostType, self._hostLocation, self._hostUsername, self._hostPassword ) + self._vms.append( vm ) + #TODO catch/transform exceptions + + def addTestCase( self, path ): + self._testcases.append( testcase.TestCase( path ) ) + + + def run( self ): + while True: + try: + inst = self._source.nextInstaller() + if inst == None: + print( "** Installer source returned None, aborting" ) + return + if inst.error: + raise ControlException( inst.error ) + print( "** New installer: {0}".format( inst.path ) ) + self.testInstaller( inst, inst.platform ) + except KeyboardInterrupt: + raise + except: + self._reporter.reportException() + + def testInstaller( self, inst, platform ): + for vm in self._vms: + if vm.ostype() != platform: + continue + for case in self._testcases: + if not case.supportsPlatform( platform ): + continue + res = result.Result() + try: + try: + res.setInstaller( inst ) + res.setTestCase( case ) + res.setVirtualMachine( vm ) + res.testStarted() + self.run_test( inst.path, vm, case, res ) + res.testFinished() + inst.markAsTested() + except KeyboardInterrupt: + raise + except: + res.addException() + finally: + self._reporter.reportResult( res ) + + def convertCheckerResults( self, filename, res ): + parser = make_parser() + parser.setContentHandler( Handler( res ) ) + f = file( filename, 'rb' ) + parser.parse( f ) + + def run_test( self, installerPath, vm, testcase, res ): + steps = testcase.steps() + if len( steps ) == 0: + raise ControlException( "No steps found for testcase {0}".format( testcase.name() ) ) + + revertStatus, _ = vm.revertToSnapshot() + if revertStatus != 0: + raise VMException( "Failed to revert to snapshot '{0}'".format( vm.snapshot() ) ) + + + time.sleep( 5 ) # Trying to avoid a possible race between restore and start + + vm.start() + + try: + try: + vm.checkPythonInstalled() + wrapperpath = vm.copyToTemp( utils.execution_path( 'guest.py' ) ) + + for stepNum in range( len( steps ) ): + needSnapshot = False + step = steps[stepNum] + if stepNum == 0: + executableguestpath = vm.copyToTemp( installerPath ) + else: + executableguestpath = testcase.maintenanceToolLocation() + + outputFileName = 'output{0}.log'.format( stepNum ) + outputpath = vm.mkTempPath( outputFileName ) + scriptguestpath = vm.copyToTemp( step.installscript() ) + timeout = step.timeout() + checkerguestpath = vm.copyToTemp( step.checkerTestDir(), "checkerTestDir{0}".format( stepNum ) ) if len( string.strip( step.checkerTestDir() ) ) > 0 else None + vm.command( 'Execute installer', "runProgramInGuest", "'{0}' '{1}' '{2}' '{3}' '{4}' --script '{5}'".format( vm.python(), wrapperpath, outputpath, timeout, executableguestpath, scriptguestpath ) ) + vm.copyFromTemp( outputFileName, outputFileName ) + r = ConfigParser.SafeConfigParser() + r.read( outputFileName ) + try: + s = r.get( 'Result', 'ExitCode' ) + exitCode = int( s ) + except ValueError: + res.addInternalError( "Could not parse integer exit code from '{0}'".format( r.get( 'Result', 'ExitCode' ) ) ) + exitCode = -1 + try: + s = r.get( 'Result', 'ExecutionTime' ) + executionTime = float( s ) + except ValueError: + res.addInternalError( "Could not parse float execution time from '{0}'".format( r.get( 'Result', 'ExecutionTime' ) ) ) + executionTime = 0.0 + + exitStatus = result.exitStatusFromString( r.get( 'Result', 'ExitStatus' ) ) + instR = result.ExecutionResult( exitCode, exitStatus, executionTime ) + + if instR.hasError(): + needSnapshot = True + + checkerResults = [] + if checkerguestpath and not instR.hasError(): + if ( platform.system() == "Darwin" ): + # Have to sleep to work around VMware Fusion bug + time.sleep( 30 ) + run_py = vm.copyToTemp( self._checkerDir ) + vm.pathSep() + "run.py" + if ( platform.system() == "Darwin" ): + # Have to sleep to work around VMware Fusion bug + time.sleep( 30 ) + checkeroutputFileName = 'checker-output{0}.xml'.format( stepNum ) + checkeroutput = vm.mkTempPath( checkeroutputFileName ) + vm.command( 'Execute checker tests', "runProgramInGuest", "'{0}' '{1}' '{2}' -o '{3}' -p '{4}'".format( vm.python(), run_py, checkerguestpath, checkeroutput, testcase.targetDirectory() ) ) + vm.copyFromTemp( checkeroutputFileName, checkeroutputFileName ) + self.convertCheckerResults( localcheckeroutput, checkerResults ) + if res.hasCheckerErrors(): + needSnapshot = True + if self._createErrorSnapshots and needSnapshot: + snapshot = 'error-{0}-{1}'.format( datetime.datetime.now().strftime( '%Y%m%d_%H%M%S' ), utils.randomString( 4 ) ) + status, _ = vm.createSnapshot( snapshot ) + if status == 0: + res.setErrorSnapshot( snapshot ) + else: + res.addInternalError( 'Could not create error snapshot "{0}"'.format( snapshot ) ) + res.addStepResult( result.StepResult( instR, checkerResults ) ) + #TODO handle timeouts? + finally: + vm.kill() + except e: + print( e ) + res.addInternalError( str( e ) ) + \ No newline at end of file diff --git a/tests/test-framework/vmware/ftpsource.py b/tests/test-framework/vmware/ftpsource.py new file mode 100644 index 000000000..306c76b63 --- /dev/null +++ b/tests/test-framework/vmware/ftpsource.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +from ftplib import FTP +import datetime, functools, os, time, tempfile +from source import Installer +import utils + +def _timestampFromFilename( platform, fn ): + # windows: assumed format is YYYY_mm_dd_HH_MM as prefix + # linux: assumed format is YYYY-mm-dd as suffix + if platform.startswith( 'windows' ): + format = '%Y_%m_%d_%H_%M' + length = len( 'YYYY_mm_dd_HH_MM' ) + return datetime.datetime.strptime( fn[0:length], format ) + else: + format = '%Y-%m-%d' + length = len( 'YYYY-mm-dd' ) + return datetime.datetime.strptime( fn[-length:], format ) + + + +def timestampFromFilename( platform, fn ): + try: + return _timestampFromFilename( platform, fn ) + except ValueError: + return None + +def filesNewerThan( files, dt ): + if dt: + return filter( lambda (x,y): y >= dt, files ) + else: + return files + +#internal to FtpSource +class Location: + def __init__( self, host, path, platform ): + self.host = host + self.path = path + self.platform = platform + self.testedFiles = [] + + def ls( self ): + ftp = FTP( self.host ) + ftp.login() + try: + return map( utils.basename, ftp.nlst( self.path ) ) + finally: + ftp.close() + + def filesSortedByTimestamp( self ): + files = self.ls() + withTS = [( i, timestampFromFilename( self.platform, i ) ) for i in files] + filtered = filter( lambda (x,y): y != None, withTS ) + filtered.sort( key=lambda (x,y): y ) + return filtered + + def untestedFilesSortedByTimestamp( self ): + l = self.filesSortedByTimestamp() + return filter( lambda ( x, y ): not self.isTested( x ), l ) + + def markAsTested( self, filename ): + self.testedFiles.append( filename ) + + def isTested( self, filename ): + return filename in self.testedFiles + + def description( self ): + return "host={0} path={1} platform={2}".format( self.host, self.path, self.platform ) + + def download( self, fn, target ): + ftp = FTP( self.host ) + ftp.login() + try: + ftp.retrbinary( 'RETR {0}/{1}'.format( self.path, fn ), target.write ) + finally: + ftp.close() + +class FtpSource: + def __init__( self, delay=60*60, tempdir='/tmp' ): + self._locations = [] + self._delay = delay + self._tempdir = tempdir + self._startDate = None + + def setStartDate( self, date ): + self._startDate = date + + def addLocation( self, host, path, platform ): + self._locations.append( Location( host, path, platform ) ) + + def nextInstaller( self ): + while True: + for i in self._locations: + print( "** Checking FTP location: " + i.description() ) + files = i.untestedFilesSortedByTimestamp() + if self._startDate != None: + files = filesNewerThan( files, self._startDate ) + if len( files ) == 0: + continue; + fn, ts = files[0] + print( "** Downloading new installer: {0}...".format( fn ) ) + tmp = tempfile.NamedTemporaryFile( dir=self._tempdir, prefix=fn ) + i.download( fn, tmp ) + print( "** Download completed. ({0})".format( tmp.name ) ) + i.lastTested = ts + inst = Installer( tmp.name, i.platform, i, ts, tmp ) + inst.sourceFilename = fn + return inst + print( "** No installers found. Going to sleep for {0} seconds...".format( self._delay ) ) + time.sleep( self._delay ) + +if __name__ == "__main__": + src = FtpSource() + src.addLocation( "hegel", "/projects/ndk/installers/windows", "windows" ) + while True: + inst = src.nextInstaller() + print inst + inst.markAsTested() diff --git a/tests/test-framework/vmware/guest.py b/tests/test-framework/vmware/guest.py new file mode 100644 index 000000000..b410d366b --- /dev/null +++ b/tests/test-framework/vmware/guest.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import sys, subprocess, ConfigParser, platform, os + +# begin Autobuild/helpers/runcommand.py + +import os +import re +import time +import sys +import signal +from subprocess import Popen +from threading import Thread +from subprocess import PIPE +import subprocess +import platform + +def DebugN( l, m ): + pass + +def windowskill( pid ): + """ replace os.kill on Windows, where it is not available""" + Cmd = 'taskkill /PID ' + str( int( pid ) ) + ' /T /F' + if os.system( Cmd ) == 0: + DebugN( 4, 'windowskill: process ' + str( pid ) + ' killed.' ) + +def kill( pid, signal ): + if 'Windows' in platform.platform(): + windowskill( pid ) + else: + os.kill( pid, signal ) + +class CommandRunner( Thread ): + def __init__ ( self, Cmd ): + Thread.__init__( self ) + self.__started = None + self.mCmd = Cmd + self.mOutput = () + self.mPid = -1 + self.mReturnCode = -1 + self.mError = () + self.__combinedOutput = False + + def setCombinedOutput(self, combine): + if combine: # make sure combine is usable as a boolean + self.__combinedOutput = True + else: + self.__combinedOutput = False + + def getCombineOutput(self): + return self.__combinedOutput + + def run( self ): + DebugN( 4, 'RunCommand: ' + str( self.mCmd ) ) + stderrValue = subprocess.PIPE + if self.__combinedOutput: + stderrValue = subprocess.STDOUT + self.__started = True +#nokia-sdk: changed to shell=False, as shell=True requires manual quoting of args + p = Popen ( self.mCmd, shell = False, stdout=subprocess.PIPE, stderr=stderrValue ) + self.mPid = p.pid + self.mOutput, self.mError = p.communicate() + self.mReturnCode = p.returncode + DebugN( 4, 'ReturnCode of RunCommand: ' + str( p.returncode ) ) + + def started(self): + return self.__started + + def output( self ): + return self.mOutput, self.mError + + def terminate( self ): + # FIXME logic? + if self.mPid != -1: + kill( self.mPid, signal.SIGTERM ) + if self.mPid == True: + self.join( 5 ) + kill( self.mPid, signal.SIGKILL ) + self.mPid = -1 + + +def RunCommand( Cmd, TimeOutSeconds=-1, CombineOutput = False ): + timeoutString = ' without a timeout' + if TimeOutSeconds > 0: + timeoutString = ' with timeout of ' + str( TimeOutSeconds ) + combinedOutputString = ' and separate output for stdout and stderr' + if CombineOutput: + combinedOutputString = ' and combined stdout and stderr output' + DebugN ( 3, 'RunCommand: executing ' + str( Cmd ) + timeoutString + combinedOutputString ) + runner = CommandRunner ( Cmd ) + runner.setCombinedOutput( CombineOutput ) + runner.start() + # poor man's mutex: + while not runner.started(): + time.sleep( 0.1 ) +# if "CYGWIN" in platform.platform() or 'Windows' in platform.platform(): +# time.sleep( 1 ) + if TimeOutSeconds == -1: + runner.join() + else: + runner.join( TimeOutSeconds ) + if runner.isAlive(): + runner.terminate() +# if "CYGWIN" in platform.platform() or 'Windows' in platform.platform(): +# time.sleep(1) + runner.join( 5 ) + DebugN( 3, 'RunCommand: command timed out, returncode is ' + str( runner.mReturnCode ) ) + return ( runner.mReturnCode, runner.output(), True ) + else: + DebugN( 3, 'RunCommand: command completed, returncode is ' + str( runner.mReturnCode ) ) + return ( runner.mReturnCode, runner.output(), False ) + +# end Autobuild/helpers/runcommand.py + + +def printUsage(): + print( "Usage: {0} [*]".format( sys.argv[0] ) ) + +if len( sys.argv ) < 4: + printUsage() + sys.exit( 1 ) + +output = sys.argv[1] +timeout = int( sys.argv[2] ) +cmd = sys.argv[3:] + +start = time.clock() +exitCode, pout, timedOut = RunCommand( cmd, timeout ) +end = time.clock() + +if timedOut: + exitStatus = 'Timeout' +else: + exitStatus = 'Normal' #TODO: detect crash + +config = ConfigParser.SafeConfigParser() +config.add_section( 'Result' ) +config.set( 'Result', 'ExitCode', str( exitCode ) ) +config.set( 'Result', 'ExitStatus', exitStatus ) +config.set( 'Result', 'Filename', cmd[0] ) +config.set( 'Result', 'Arguments', " ".join( cmd[1:] ) ) +config.set( 'Result', 'Timeout', str( timeout ) ) +config.set( 'Result', 'ExecutionTime', str( end - start ) ) +config.set( 'Result', 'Stdout', pout[0] ) +config.set( 'Result', 'Stderr', pout[1] ) + +config.add_section( 'Platform' ) +config.set( 'Platform', 'system', platform.system() ) +config.set( 'Platform', 'release', platform.release() ) +config.set( 'Platform', 'version', platform.version() ) +config.set( 'Platform', 'machine', platform.machine() ) + +with open( output, 'w' ) as configFile: + config.write(configFile ) + +sys.exit( exitCode ) + + + + diff --git a/tests/test-framework/vmware/guestconfig.py b/tests/test-framework/vmware/guestconfig.py new file mode 100644 index 000000000..f80b1c73d --- /dev/null +++ b/tests/test-framework/vmware/guestconfig.py @@ -0,0 +1,11 @@ +import os, ConfigParser, tempfile, platform +config = ConfigParser.SafeConfigParser() +config.add_section('Guest') +config.set('Guest', 'temp_dir', str(tempfile.gettempdir())) +config.set('Guest', 'path_sep', os.sep) +config.set('Guest', 'system', platform.system()) +config.set('Guest', 'release', platform.release()) +config.set('Guest', 'version', platform.version()) +config.set('Guest', 'machine', platform.machine()) +with open('example.cfg', 'wb') as configFile: + config.write(configFile) diff --git a/tests/test-framework/vmware/reporter.py b/tests/test-framework/vmware/reporter.py new file mode 100644 index 000000000..57366a8fe --- /dev/null +++ b/tests/test-framework/vmware/reporter.py @@ -0,0 +1,96 @@ +import datetime, socket, sys, traceback +from xml.sax.saxutils import XMLGenerator +from xml.sax.xmlreader import AttributesNSImpl +import result +from result import exitStatusAsString +from xmlutils import startElement, endElement, writeElement + +class Reporter: + def __init__( self ): + pass + + def reportException( self ): + sys.stderr.write( traceback.format_exc( 15 ) + '\n' ) + + def reportResult( self, result ): + try: + self.toXml( result, sys.stdout ) + finally: + sys.stdout.flush() + + def toXml( self, result, out ): + atom = "http://www.w3.org/2005/Atom" + tf = "http://sdk.nokia.com/test-framework/ns/1.0" + + gen = XMLGenerator( out, 'utf-8' ) + gen.startDocument() + gen.startPrefixMapping( 'atom', atom) + gen.startPrefixMapping( 'tf', tf ) + + startElement( gen, atom, 'entry' ) + + writeElement( gen, atom, 'title', result.constructTitle() ) + writeElement( gen, atom, 'updated', datetime.datetime.now().isoformat() ) + + writeElement( gen, tf, 'errorSummary', exitStatusAsString( result.status() ) ) + + writeElement( gen, tf, 'host', socket.gethostname() ) + if result._testStart != None: + writeElement( gen, tf, 'testStart', result._testStart.isoformat() ) + else: + result._internalErrors.append( "Result generator: no start timestamp found." ) + + if result._testEnd != None: + writeElement( gen, tf, 'testEnd', result._testEnd.isoformat() ) + else: + result._internalErrors.append( "Result generator: no end timestamp found." ) + + + startElement( gen, tf, 'installer' ) + writeElement( gen, tf, 'sourceUrl', result._installerSourceLocation ) + writeElement( gen, tf, 'platform', result._installerTargetPlatform ) + #TODO revision + endElement( gen, tf, 'installer' ) + + if result._testcase != None: + startElement( gen, tf, 'testCase' ) + writeElement( gen, tf, 'name', result._testcase.name() ) + writeElement( gen, tf, 'path', result._testcase.path() ) + writeElement( gen, tf, 'installScript', result._testcase.installscript() ) + endElement( gen, tf, 'testCase' ) + else: + result._internalErrors.append( "Result generator: No test case given." ) + + if result._installationResult != None: + startElement( gen, tf, 'installationResult' ) + writeElement( gen, tf, 'exitCode', str( result._installationResult.exitCode ) ) + writeElement( gen, tf, 'exitStatus', exitStatusAsString( result._installationResult.exitStatus ) ) + endElement( gen, tf, 'installationResult' ) + else: + result._internalErrors.append( "Result generator: No installation result given." ) + startElement( gen, tf, 'checkerResult' ) + + for err in result._checkerErrors: + writeElement( gen, tf, 'error', err ) + endElement( gen, tf, 'checkerResult' ) + + startElement( gen, tf, 'virtualMachine' ) + writeElement( gen, tf, 'path', result._vm.vmxPath() ) + writeElement( gen, tf, 'platform', result._vm.ostype() ) + writeElement( gen, tf, 'snapshot', result._vm.snapshot() ) + endElement( gen, tf, 'virtualMachine' ) + + startElement( gen, tf, 'internalErrors' ) + for i in result._internalErrors: + writeElement( gen, tf, 'internalError', str( i ) ) + + endElement( gen, tf, 'internalErrors' ) + + if result._errorSnapshot != None: + writeElement( gen, tf, 'errorSnapshot', result._errorSnapshot ) + + endElement( gen, atom, 'entry' ) + + gen.endDocument() + + diff --git a/tests/test-framework/vmware/result.py b/tests/test-framework/vmware/result.py new file mode 100644 index 000000000..07d4832aa --- /dev/null +++ b/tests/test-framework/vmware/result.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +import control, testcase, datetime, source, tempfile, traceback, utils +from source import Installer + +class ExecutionResult: + #return codes: + Success = 0 + InstallationFailed = 1 + InstallationCanceled = 2 + + #exit status: + Normal = 0 + Crash = 1 + Timeout = 2 + + def __init__( self, exitCode, exitStatus, executionTime ): + self.exitCode = exitCode + self.exitStatus = exitStatus + self.executionTime = executionTime + + def hasError( self ): + return self.exitCode != 0 or self.exitStatus != ExecutionResult.Normal + +class CheckerResult: + Passed = 0 + Failed = 1 + NotRun = 2 + + def hasError( list ): + #the following would more efficient with something find_if-like (stopping if test with error is found) + return len( [x for i in list if i.hasError() ] ) > 0 + + def __init__( self, name, result, errorString ): + self.name = utils.randomString( 3 ) + '_' + name + self.result = result + self.errorString = errorString + + def hasError( self ): + return self.result != CheckerResult.Passed + +class StepResult: + def __init__( self, executionResult, checkerResults ): + self.executionResult = executionResult + self.checkerResults = checkerResults + + def hasCheckerErrors( self ): + return CheckerResult.hasError( self.checkerResults ) + +def exitStatusFromString( s ): + if s == 'Crash': + return ExecutionResult.Crash + if s == 'Timeout': + return ExecutionResult.Timeout + if s == 'Normal': + return ExecutionResult.Normal + raise control.ControlException( "Unknown exit status string: {0}".format( s ) ) + +#there is probably a cooler way with introspection and stuff: +def exitStatusAsString( status ): + if status == ExecutionResult.Crash: + return 'Crash' + if status == ExecutionResult.Normal: + return 'Normal' + if status == ExecutionResult.Timeout: + return 'Timeout' + raise control.ControlException( "Unknown exit status: {0}".format( status ) ) + +class Result: + + NoError=0 # Everything ok + InstallerError=1 #Installer failed, or post-conditions not met + InternalError=2 #Internal test framework error + + def __init__( self ): + self._internalErrors = [] + self._stepResults = [] + self._installer = None + self._testStart = None + self._testEnd = None + self._testcase = None + self._vm = None + self._errorSnapshot = None + self._revision = "revision-todo" + + def setInstaller( self, installer ): + self._installer = installer + + def testStarted( self ): + self._testStart = datetime.datetime.now() + + def testFinished( self ): + self._testEnd = datetime.datetime.now() + + def setTestCase( self, testcase ): + self._testcase = testcase + + def setErrorSnapshot( self, name ): + self._errorSnapshot = name + + def setVirtualMachine( self, vm ): + self._vm = vm + + def addInternalError( self, errstr ): + self._internalErrors.append( errstr ) + + def hasInternalErrors( self ): + return len( self._internalErrors ) > 0 + + def addException( self ): + s = 'Unexpected exception: {0}'.format( traceback.format_exc( 15 ) ) + self._internalErrors.append( s ) + + def hasCheckerErrors( self ): + for step in self._stepResults: + if CheckerResult.hasError( step.checkerResults ): + return True + return False + + + def addStepResult( self, result ): + self._stepResults.append( result ) + + def status( self ): + if len( self._internalErrors ) > 0: + return Result.InternalError + if len( self._stepResults ) == 0: + return Result.InternalError + for step in self._stepResults: + if step.executionResult.exitCode != ExecutionResult.Success or step.exitStatus != ExecutionResult.Normal: + return Result.InstallerError + if step.hasCheckerErrors(): + return Result.InstallerError + #TODO: check test results + return Result.NoError + + def statusAsNiceString( self ): + s = self.status() + if s == Result.InternalError: + return "Internal framework errors" + if s == Result.InstallerError: + return "Installation error" + return "OK" + + def constructTitle( self ): + smiley = ":-)" if self.status() == Result.NoError else ":-(" + + return "{0}: {1} on {2} testing {3} - {4} {5}".format( self._revision, utils.basename( self._installerSourceLocation ), self._vm.name(), self._testcase.name(), self.statusAsNiceString(), smiley ) + \ No newline at end of file diff --git a/tests/test-framework/vmware/run-test.py b/tests/test-framework/vmware/run-test.py new file mode 100644 index 000000000..c063f3e70 --- /dev/null +++ b/tests/test-framework/vmware/run-test.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +import sys, os, ConfigParser, optparse, time, virtualmachine, utils + +# TODO: - Look in sensible locations for python in the guest VM +# - Try and work-around VMware Fusion needing to be closed + + +def check_option( option, optionName ): + if option: + return + print("** Could not find a value for {0}".format(optionName) ) + print("** Please specify it on the commandline or configuration file.") + print + optionParser.print_help(); + sys.exit(-1) + +def die(message): + print "** " + str(message) + sys.exit(1); + +# +# Parse options +# + +optionParser = optparse.OptionParser(usage="%prog [options] vmx-file", version="%prog 0.9") +optionParser.add_option("-u", "--username", dest="username", help="username for VM", metavar="USERNAME" ) +optionParser.add_option("-p", "--password", dest="password", help="password for VM", metavar="PASSWORD" ) +optionParser.add_option("-S", "--script", dest="script", help="script for VM", metavar="SCRIPT" ) +optionParser.add_option("-s", "--snapshot", dest="snapshot", help="snapshot for VM", metavar="SNAPSHOT" ) +optionParser.add_option("-P", "--python", dest="python", help="python location in VM", metavar="PYTHON" ) +optionParser.add_option("-v", "--vmrun", dest="vmrun", help="vmrun command for VM", metavar="VMRUN") +optionParser.add_option("-c", "--config", dest="config", help="configuration file for VM", metavar="CONFIG") +optionParser.add_option("-i", "--installer", dest="installer", help="installer executable to command inside the VM", metavar="INSTALLER") +optionParser.add_option("-e", "--installscript", dest="installscript", help="QtScript script to use for non-interactive installation", metavar="INSTALLSCRIPT") + +(options, args) = optionParser.parse_args() + +try: + options.vmx = args[0] +except IndexError: + options.vmx = None + +options.ostype = None +options.guestTempDir = None + +config = ConfigParser.SafeConfigParser() +if options.config: + print("** Reading config: " + options.config ) + config.read( options.config ) +else: + print("** No config given") + +options.username = utils.get_config_option( config, options.username, "username", "nokia" ) +options.password = utils.get_config_option( config, options.password, "password", "nokia" ) +options.script = utils.get_config_option( config, options.script, "script", "guest.py" ) +options.python = utils.get_config_option( config, options.python, "python", "c:/python26/python.exe" ) +options.snapshot = utils.get_config_option( config, options.snapshot, "snapshot", "base" ) +options.vmrun = utils.get_config_option( config, options.vmrun, "vmrun" ) +options.vmx = utils.get_config_option( config, options.vmx, "vmx" ) +options.guestTempDir = utils.get_config_option( config, options.guestTempDir, "tempDir", "c:\\windows\\temp" ) +options.ostype = utils.get_config_option( config, options.ostype, "os", "windows" ) + +check_option( options.vmx, "vmx-file" ) + +# Search the PATH and the a few extra locations for 'vmrun' +if not options.vmrun: + options.vmrun = utils.findVMRun() + +check_option( options.vmrun, "VMRUN" ) +check_option( options.installer, "INSTALLER" ) +check_option( options.installscript, "INSTALLSCRIPT" ) + +# +# VM actions +# + +vm = virtualmachine.VirtualMachine( options.vmrun, options.vmx, options.username, options.password, options.guestTempDir, options.ostype ) + +snapshotExists = vm.snapshotExists( options.snapshot ) +if not snapshotExists: + die("Could not find '{0}' snapshot, please create it in the VM.".format( options.snapshot ) ) + +revertStatus, _ = vm.revertToSnapshot( options.snapshot ) +if revertStatus != 0: + die("Failed to revert to snapshot") + +time.sleep( 5 ) # Trying to avoid a possible race between restore and start + +vm.start() + +try: + pythonStatus, _ = vm.command("Checking for guest installed Python", "fileExistsInGuest", options.python) + if pythonStatus != 0: + raise virtualmachine.VMException("Please install Python in the VM from http://www.python.org/download/") + + wrapperpath = vm.copyToTemp( 'guest.py' ) + installerpath = vm.copyToTemp( options.installer ) + scriptpath = vm.copyToTemp( options.installscript ) + outputpath = vm.mkTempPath( 'output.log' ) + vm.command( 'Execute installer', "runProgramInGuest", "-interactive -activeWindow '{0}' '{1}' '{2}' '{3}' --script '{4}'".format( options.python, wrapperpath, outputpath, installerpath, scriptpath ) ) + + vm.copyFromTemp( 'output.log', 'output.log' ) + result = ConfigParser.SafeConfigParser() + result.read( 'output.log' ) + print( "Installer exit code: " + result.get( 'Result', 'ExitCode' ) ) + #TODO parse output and do something with it + +except virtualmachine.VMException as exception: + die( exception ) +finally: + vm.kill() diff --git a/tests/test-framework/vmware/run.py b/tests/test-framework/vmware/run.py new file mode 100644 index 000000000..f02571051 --- /dev/null +++ b/tests/test-framework/vmware/run.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import ConfigParser, datetime, optparse, os, sys +from functools import partial +import cdashreporter, control, ftpsource, source, reporter, utils + +def die( msg ): + sys.stderr( msg + '\n' ) + sys.exit( 1 ) + +optionParser = optparse.OptionParser(usage="%prog [options] configfile installer0 [installer1 ...]", version="%prog 0.1") +optionParser.add_option("-r", "--vmrun", dest="vmrun", help="vmrun executable to use", metavar="VMRUN" ) +optionParser.add_option("-s", "--only-since", dest="since", help="test only installers newer than timestamp (YYYY-MM-DD or YYYY-MM-DD-hh-mm)", metavar="SINCE" ) +optionParser.add_option("-c", "--checkerInstallation", dest="checkerInstallation", help="checker installation to use for post-installation checks", metavar="CHECKERINSTALLATION" ) +(options, args) = optionParser.parse_args() + +try: + configpath = utils.makeAbsolutePath( args[0], os.getcwd() ) +except IndexError: + optionParser.print_usage( sys.stderr ) + sys.exit( 1 ) + +config = ConfigParser.SafeConfigParser() +config.read( configpath ) + +#make an unary functor to create absolute paths +abspath = partial( utils.makeAbsolutePath, relativeTo=os.path.dirname( configpath ) ) + +vmrun = utils.get_config_option( config, options.vmrun, "vmrun", utils.findVMRun() ) +useGui = utils.get_config_option( config, None, "gui", "true" ).lower() == "true" +createErrorSnapshots = utils.get_config_option( config, None, "createErrorSnapshots", "true" ).lower() == "true" +hostType = utils.get_config_option( config, None, "type", "", "Host" ) +hostLocation = utils.get_config_option( config, None, "location", "", "Host" ) +hostUsername = utils.get_config_option( config, None, "username", "", "Host" ) +hostPassword = utils.get_config_option( config, None, "password", "", "Host" ) + +cdashHost = utils.get_config_option( config, None, "host", "", "CDash" ) +cdashLocation = utils.get_config_option( config, None, "location", "", "CDash" ) +cdashProject = utils.get_config_option( config, None, "project", "", "CDash" ) + +if vmrun == None: + die( "Could not find vmrun executable. Please specify it in the config file (vmrun=...) or via the --vmrun option" ) + +checkerInstallation = utils.get_config_option( config, options.checkerInstallation, "checkerInstallation" ) +if checkerInstallation == None: + die( "Could not find checker installation. Please specify it in the config file (checkerInstallation=...) or via the --checkerInstallation option" ) + +#apply functor to list to get absolute paths: +testcases = map( abspath, utils.get_enumerated_config_option( config, 'testcase' ) ) +if len( testcases ) == 0: + die( "No testcases specified. Please specify at least one test case in the configuration" ) + +vms = map( abspath, utils.get_enumerated_config_option( config, 'vm' ) ) +if len( vms ) == 0: + die( "No VMs specified. Please specify at least one VM in the configuration" ) + +installers = args[1:] + +if len( installers ) > 0: + source = source.Source() + for i in installers: + source.addDummy( 5, i ) +else: + source = ftpsource.FtpSource() + if options.since: + try: + sdt = datetime.datetime.strptime( options.since, '%Y-%m-%d' ) + except ValueError: + sdt = datetime.datetime.strptime( options.since, '%Y-%m-%d-%H-%M' ) + source.setStartDate( sdt ) + found = True + nextsec = 0 + while found: + sec = "Source{0}".format( nextsec ) + nextsec += 1 + try: + host = config.get( sec, "host" ) + path = config.get( sec, "path" ) + platform = config.get( sec, "platform" ) + print( "** Add FTP location {0}:{1} ({2})".format( host, path, platform ) ) + source.addLocation( host, path, platform ) + except ConfigParser.NoSectionError: + found = False + +cdashHost = utils.get_config_option( config, None, "host", "", "CDash" ) +cdashLocation = utils.get_config_option( config, None, "location", "", "CDash" ) +cdashProject = utils.get_config_option( config, None, "project", "", "CDash" ) + +if len( cdashHost ) > 0: + reporter = cdashreporter.CDashReporter( cdashHost, cdashLocation, cdashProject ) +else: + reporter = reporter.Reporter() + +control = control.Control( vmrun, checkerInstallation, source, reporter ) +control.setGuiEnabled( useGui ) +control.setCreateErrorSnapshots( createErrorSnapshots ) + +if len( hostType ) > 0: + control.setRemoteHost( hostType, hostLocation, hostUsername, hostPassword ) + +for i in vms: + control.addVM( i ) +for i in testcases: + control.addTestCase( i ) + +control.run() diff --git a/tests/test-framework/vmware/source.py b/tests/test-framework/vmware/source.py new file mode 100644 index 000000000..fe067ab4e --- /dev/null +++ b/tests/test-framework/vmware/source.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +import os, time + +class Installer: + def __init__( self, path, platform, location, timestamp=None, tempfile=None ): + self.path = path + self.platform = platform + self.error = None + self.timestamp = timestamp + self.tempfile = tempfile + self.sourceLocation = location + + def markAsTested( self ): + self.sourceLocation.markAsTested( self.sourceFilename ) + +class Source: + def __init__( self ): + self._dummies = [] + + def nextInstaller( self ): + if len( self._dummies ) == 0: + return None + delay, path = self._dummies.pop() + time.sleep( delay ) + if os.path.exists( path ): + inst = Installer( path, "linux", None ) + inst.sourceFilename = path + return inst + else: + inst = Installer( None, None ) + #simulating download errors + inst.error = "Installer '{0}' does not exist".format( path ) + return inst + + def addDummy( self, delay, path ): + self._dummies.insert( 0, ( delay, path ) ) diff --git a/tests/test-framework/vmware/testcase.py b/tests/test-framework/vmware/testcase.py new file mode 100644 index 000000000..65fd83880 --- /dev/null +++ b/tests/test-framework/vmware/testcase.py @@ -0,0 +1,70 @@ +import ConfigParser, os, utils + +class Step: + def __init__( self, installscript, checkerTestDir, timeout ): + self._installscript = installscript + self._checkerTestDir = checkerTestDir + self._timeout = timeout + + def installscript( self ): + return self._installscript + + def checkerTestDir( self ): + return self._checkerTestDir + + def timeout( self ): + return self._timeout + +class TestCase: + def __init__( self, path ): + self._steps = [] + config = ConfigParser.SafeConfigParser() + config.read( path ) + self._path = path + self._platforms = [] + found = True + stepNum = 0 + while found: + sec = "Step{0}".format( stepNum ) + if not config.has_section( sec ): + found = False + continue + + stepNum += 1 + installscript = utils.makeAbsolutePath( utils.get_config_option( config, None, "installscript", None, sec ), os.path.dirname( path ) ) + checkerTestDir = utils.get_config_option( config, None, "checkerTestDir", None, sec ) + checkerTestDir = utils.makeAbsolutePath( checkerTestDir, os.path.dirname( path ) ) if checkerTestDir else "" + timeout = int( utils.get_config_option( config, None, "timeout", 60 * 60, sec ) ) + self._steps.append( Step( installscript, checkerTestDir, timeout ) ) + + self._name = utils.get_config_option( config, None, "name", utils.basename( path ) ) + self._targetDirectory = utils.get_config_option( config, None, "targetDirectory", "" ) + self._maintenanceToolLocation = utils.get_config_option( config, None, "maintenanceToolLocation", "" ) + platforms = utils.get_config_option( config, None, "platforms" ) + if platforms != None: + self._platforms = platforms.split( ',' ) + + def supportsPlatform( self, platform ): + return platform in self._platforms + + def installerTimeout( self ): + return self._installerTimeout + + def platforms( self ): + return self._platforms + + def name( self ): + return self._name + + def targetDirectory( self ): + return self._targetDirectory + + def steps( self ): + return self._steps + + def path( self ): + return self._path + + def maintenanceToolLocation( self ): + return self._maintenanceToolLocation + \ No newline at end of file diff --git a/tests/test-framework/vmware/utils.py b/tests/test-framework/vmware/utils.py new file mode 100644 index 000000000..24529be40 --- /dev/null +++ b/tests/test-framework/vmware/utils.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +import ConfigParser, inspect, os, string, sys +from random import Random + +def findVMRun(): + searchDirectories = os.environ['PATH'].split(os.pathsep) + searchDirectories.append("/Library/Application Support/VMware Fusion") + for directory in searchDirectories: + possibleFile = os.path.join(directory, 'vmrun') + if os.path.isfile(possibleFile): + return possibleFile + return None + +def basename( path ): + if path.endswith( os.path.sep ): + return os.path.basename( path[0,-1] ) + else: + return os.path.basename( path ) + +def makeAbsolutePath( path, relativeTo ): + if os.path.isabs( path ) or relativeTo == None: + return path + else: + return relativeTo + os.sep + path + +def execution_path( filename ): + return os.path.join( os.path.dirname( inspect.getfile( sys._getframe( 1 ) ) ), filename ) + +def unixPathSep( s ): + return s.replace( '\\', '/' ) + +def get_config_option( config, initial, option, default=None, section="General" ): + # If there's already a valid option from the commandline, + # override the value from the configuration file. + if initial != None: + print( "** Overridden {0}={1}".format( option, initial ) ) + return initial + try: + tmp = config.get( section, option ) + print( "** Read config value {0}={1}".format( option, tmp ) ) + return tmp + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + print( "** Using default value {0}={1}".format( option, default ) ) + return default + +def get_enumerated_config_option( config, option ): + res = [] + i = 0 + while True: + key = option + str( i ) + val = get_config_option( config, None, key ) + if val == None: + print res + return res + res.append( val ) + i += 1 + +def randomString( length ): + return ''.join( Random().sample( string.letters + string.digits, length ) ) diff --git a/tests/test-framework/vmware/virtualmachine.py b/tests/test-framework/vmware/virtualmachine.py new file mode 100644 index 000000000..7507cd531 --- /dev/null +++ b/tests/test-framework/vmware/virtualmachine.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +import commands, ConfigParser, os, string, utils, platform + +class VMException(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class VirtualMachine: + def __init__( self, vmrun, vmx, username, password, tempDir, ostype ): + self._vmrun = vmrun + self._vmx = vmx + self._username = username + self._password = password + self._tempDir = tempDir + self._ostype = ostype + self._useGui = True + self._name = "no name" + self._hostType = "" + self._hostLocation = None + self._hostUsername = None + self._hostPassword = None + + def setGuiEnabled( self, usegui ): + self._useGui = usegui + + def name( self ): + return self._name + + def isRemote( self ): + return len( self._hostType ) > 0; + + def setRemoteHost( self, type, loc, user, pw ): + self._hostType = type + self._hostLocation = loc + self._hostUsername = user + self._hostPassword = pw + + + def command( self, message, command, parameters=None ): + if len( self._hostType ) > 0: + remote = "-T {0} -h '{1}' -u {2} -p {3}".format( self._hostType, self._hostLocation, self._hostUsername, self._hostPassword ) + else: + remote = "" + vmcommand = "'{0}' {1} -gu {2} -gp {3} {4} '{5}' ".format( self._vmrun, remote, self._username, self._password, command, self._vmx ) + if parameters: + vmcommand += parameters + print "** " + message + status, output = commands.getstatusoutput(vmcommand) + if status != 0: + print "* Return code: {0}".format(status) + print "* Output:" + print output + print + return status, output + + def copyFileToGuest( self, source, target ): + #TODO convert paths to guest system? + status, _ = self.command( 'Copying {0} to {1}'.format( source, target ), 'copyFileFromHostToGuest', "'{0}' '{1}'".format( source, target ) ) + if status != 0: + raise VMException( "Could not copy from host to guest: source {0} target {1}".format( source, target ) ) + + def copyFileFromGuest( self, source, target ): + #TODO convert paths to guest system? + status, _ = self.command( 'Copying {0} to {1}'.format( source, target ), 'copyFileFromGuestToHost', "'{0}' '{1}'".format( source, target ) ) + if status != 0: + raise VMException( "Could not copy from guest to host: source {0} target {1}".format( source, target ) ) + + + def ostype( self ): + return self._ostype + + def vmxPath( self ): + return self._vmx + + def mkTempPath( self, filename ): + return self._tempDir + self.pathSep() + utils.basename( filename ) + + def pathSep( self ): + if self._ostype == "windows": + return "\\" + else: + return "/" + + def copyToTemp( self, source, targetN=None ): + if source == None or len( string.strip( source ) ) == 0: + return None + if targetN == None: + targetN = source + + target = self.mkTempPath( targetN ) + self.copyFileToGuest( source, target ) + return target + + def copyFromTemp( self, filename, target ): + source = self.mkTempPath( filename ) + self.copyFileFromGuest( source, target ) + return target + + def start( self ): + arg = "gui" if self._useGui else "nogui" + self.command("Starting VM ({0})".format( arg ), "start", arg ) + + def kill( self ): + self.command("Stopping VM", "stop", "hard" ) + + def isRunning( self ): + _, vmList = self.command("Checking running VMs", "list") + _, _, vmFilename = self._vmx.rpartition( os.sep ) + return vmList.find(vmFilename) is not -1 + + def snapshotExists( self, snapshot ): + _, output = self.command("Checking snapshots", "listSnapshots") + snapshotList = output.split("\n") + # Remove first entry which contains number of snapshots + snapshotList.pop(0) + return snapshotList.count( snapshot ) > 0 + + def checkPythonInstalled( self ): + pp = utils.unixPathSep( self._python ) + pythonStatus, _ = self.command("Checking for guest installed Python", "fileExistsInGuest", pp ) + print pythonStatus + if pythonStatus != 0: + raise VMException("Could not find python in {0}: Please specify the path/install Python in the VM from http://www.python.org/download/".format( pp ) ) + else: + print("Python found ({0})".format( pp ) ) + + def python( self ): + return self._python + + def snapshot( self ): + return self._snapshot + + def revertToSnapshot( self, snapshot=None ): + if snapshot != None: + snap = snapshot + else: + snap = self._snapshot + # VMware Fusion needs to be closed before you can restore snapshots so kill it + if ( platform.system() == "Darwin" ): + commands.getstatusoutput( "ps x|grep 'VMware Fusion'|cut -d ' ' -f1|xargs kill" ) + return self.command("Reverting to '{0}' snapshot".format( snap ), "revertToSnapshot", snap ) + + def createSnapshot( self, name ): + return self.command("Creating error snapshot '{0}'".format( name ), "snapshot", name ) + + +def fromVMRunAndPath( vmrun, path ): + config = ConfigParser.SafeConfigParser() + config.read( path ) + + hostType = utils.get_config_option( config, None, "type", "", "Host" ) + hostLocation = utils.get_config_option( config, None, "location", "", "Host" ) + hostUsername = utils.get_config_option( config, None, "username", "", "Host" ) + hostPassword = utils.get_config_option( config, None, "password", "", "Host" ) + + vmxVal = utils.get_config_option( config, None, "vmx" ) + if not len( hostType ) == 0: + vmx = utils.makeAbsolutePath( vmxVal, os.path.dirname( path ) ) + else: + vmx = vmxVal + + username = utils.get_config_option( config, None, "username", "nokia" ) + password = utils.get_config_option( config, None, "password", "nokia" ) + tempDir = utils.get_config_option( config, None, "tempDir", "c:\\windows\\temp" ) + ostype = utils.get_config_option( config, None, "os", "windows" ) + + vm = VirtualMachine( vmrun, vmx, username, password, tempDir, ostype ) + vm._name = utils.get_config_option( config, None, "name", utils.basename( path ) ) + vm._snapshot = utils.get_config_option( config, None, "snapshot", "base" ) + vm._python = utils.get_config_option( config, None, "python", "c:/python26/python.exe" ) + vm._hostType = hostType + vm._hostLocation = hostLocation + vm._hostUsername = hostUsername + vm._hostPassword = hostPassword + return vm diff --git a/tests/test-framework/vmware/xmlutils.py b/tests/test-framework/vmware/xmlutils.py new file mode 100644 index 000000000..609412309 --- /dev/null +++ b/tests/test-framework/vmware/xmlutils.py @@ -0,0 +1,13 @@ +def startElement( gen, ns, name, attrsb={} ): + attrs = {} + for i in attrsb: + attrs[(None, i)] = attrsb[i] + gen.startElementNS( ( ns, name ), name, attrs ) + +def endElement( gen, ns, name ): + gen.endElementNS( ( ns, name ), name ) + +def writeElement( gen, ns, name, text, attrs={} ): + startElement( gen, ns, name, attrs ) + gen.characters( text ) + endElement( gen, ns, name ) diff --git a/tests/testcases/result-example.xml b/tests/testcases/result-example.xml new file mode 100644 index 000000000..937625b3c --- /dev/null +++ b/tests/testcases/result-example.xml @@ -0,0 +1,37 @@ + + + + ab12cd34ef - installer-20090310.exe - checks failed :-( + 2010-03-10T14:00Z + test-machine_1.local + 2010-03-10T13:00Z + 2010-03-10T13:49Z + + + ftp://buildmachine/installers/installer-20090310.exe + ab12cd34ef + windows + + + /home/testuser/VMs/WindowsXp/WindowsXp.vmx + windows + base + + + Basic Installation + /home/testuser/testcases/basic/basic.cfg + /home/testuser/testcases/basic/testscript.qs + + + 0 + normal + NoError + + + c:\sdk\bin\qmake.exe: md5 sum mismatch: expected: abcd1234 actual: 1234abcd + InstallerError + + + InstallerError + + diff --git a/tests/testcases/testcase1/packagemanagement.qs b/tests/testcases/testcase1/packagemanagement.qs new file mode 100644 index 000000000..3c6dcfdeb --- /dev/null +++ b/tests/testcases/testcase1/packagemanagement.qs @@ -0,0 +1,35 @@ +function Controller() +{ + installer.autoRejectMessageBoxes + this.componentSelectionCounter = 0 +} + +Controller.prototype.UpdaterSelectedCallback = function() +{ + tabController.setCurrentTab( TabController.PACKAGE_MANAGER ) +} + +Controller.prototype.ComponentSelectionPageCallback = function() +{ + if ( this.componentSelectionCounter == 0 ) { + print( "first time, uninstall" ) + var page = gui.pageWidgetByObjectName( "ComponentSelectionPage" ) + page.deselectComponent( "com.nokia.sdk.doc.qtcreator" ) + gui.clickButton( buttons.NextButton, 3000 ) + this.componentSelectionCounter += 1 + } else { + print( "second time, click cancel" ) + gui.clickButton( buttons.CancelButton ) + } +} + + +Controller.prototype.ReadyForInstallationPageCallback = function() +{ + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.FinishedPageCallback = function() +{ + gui.clickButton( buttons.CommitButton ) +} diff --git a/tests/testcases/testcase1/test-uninstall.qs b/tests/testcases/testcase1/test-uninstall.qs new file mode 100644 index 000000000..c2d707fe1 --- /dev/null +++ b/tests/testcases/testcase1/test-uninstall.qs @@ -0,0 +1,30 @@ +function Controller() +{ + installer.autoRejectMessageBoxes +} + + +Controller.prototype.IntroductionPageCallback = function() +{ + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.ComponentSelectionPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "ComponentSelectionPage" ) + page.keepSelectedComponentsRB.setChecked( true ) + page.deselectComponent( "com.nokia.sdk.qt.gui" ) + page.deselectComponent( "hgrmpfl (non-existing package)" ) // bad case for component lookup + page.selectComponent( "hgrmpfl2 (another non-existing package)" ) // bad case for component lookup + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.ReadyForInstallationPageCallback = function() +{ + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.FinishedPageCallback = function() +{ + gui.clickButton( buttons.FinishButton ) +} diff --git a/tests/testcases/testcase1/testcase1.cfg b/tests/testcases/testcase1/testcase1.cfg new file mode 100644 index 000000000..d66c7d56c --- /dev/null +++ b/tests/testcases/testcase1/testcase1.cfg @@ -0,0 +1,5 @@ +[General] +installscript=testscript.qs +platforms=windows,linux +targetDirectory=c:\Users\kdab\Desktop\testinstall +checkerTestDir=checker diff --git a/tests/testcases/testcase1/testscript.qs b/tests/testcases/testcase1/testscript.qs new file mode 100644 index 000000000..a19c0f9de --- /dev/null +++ b/tests/testcases/testcase1/testscript.qs @@ -0,0 +1,69 @@ +function Controller() +{ + installer.autoRejectMessageBoxes + installer.setMessageBoxAutomaticAnswer( "overwriteTargetDirectory", QMessageBox.Yes ) +} + + +Controller.prototype.IntroductionPageCallback = function() +{ + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.LicenseAgreementPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "LicenseAgreementPage" ) + page.acceptLicenseRB.setChecked( true ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.TargetDirectoryPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "TargetDirectoryPage" ) + page.targetDirectoryLE.setText( "c:\\Users\\kdab\\Desktop\\testinstall" ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.ComponentSelectionPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "ComponentSelectionPage" ) + page.deselectComponent( "com.nokia.sdk.qtcreator" ) + page.deselectComponent( "hgrmpfl (non-existing package)" ) // bad case for component lookup + page.selectComponent( "hgrmpfl2 (another non-existing package)" ) // bad case for component lookup + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.DynamicQtGuiPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "DynamicQtGuiPage" ) + page.checkBoxLib.setChecked( false ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.DynamicErrorPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "DynamicErrorPage" ) + page.checkBoxMakeSure.setChecked( true ) + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.ReadyForInstallationPageCallback = function() +{ + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.StartMenuDirectoryPageCallback = function() +{ + gui.clickButton( buttons.NextButton ) +} + +Controller.prototype.PerformInstallationPageCallback = function() +{ + var page = gui.pageWidgetByObjectName( "PerformInstallationPage" ) + page.details.button.click +} + +Controller.prototype.FinishedPageCallback = function() +{ + gui.clickButton( buttons.FinishButton ) +} diff --git a/tests/testreturn/main.cpp b/tests/testreturn/main.cpp new file mode 100644 index 000000000..82b57048e --- /dev/null +++ b/tests/testreturn/main.cpp @@ -0,0 +1,70 @@ +/************************************************************************** +** +** This file is part of Qt SDK** +** +** Copyright (c) 2011 Nokia Corporation and/or its subsidiary(-ies).* +** +** Contact: Nokia Corporation qt-info@nokia.com** +** +** No Commercial Usage +** +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this file. +** Please review the following information to ensure the GNU Lesser General +** Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception version +** 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** If you are unsure which license is appropriate for your use, please contact +** (qt-info@nokia.com). +** +**************************************************************************/ +#include +#include +#include + +static void crash() { + QString* nemesis = 0; + nemesis->clear(); +} + +int main( int argc, char** argv ) { + std::cout << "Hello." << std::endl; + if ( argc < 2 ) + return 0; + const QString arg = QString::fromLocal8Bit( argv[1] ); + bool ok = false; + const int num = arg.toInt( &ok ); + if ( ok ) + return num; + if ( arg == QLatin1String("crash") ) { + std::cout << "Yeth, mather. I Crash." << std::endl; + crash(); + } + if ( arg == QLatin1String("--script") ) { + const QString fn = QString::fromLocal8Bit( argv[2] ); + QFile f( fn ); + if ( !f.open( QIODevice::ReadOnly ) ) { + std::cerr << "Could not open file for reading: " << qPrintable(f.errorString()) << std::endl; + crash(); + } + bool ok; + const int rv = QString::fromLatin1( f.readAll() ).toInt( &ok ); + if ( ok ) + return rv; + else + crash(); + } + +} diff --git a/tests/testreturn/testreturn.pro b/tests/testreturn/testreturn.pro new file mode 100644 index 000000000..a99a57b7a --- /dev/null +++ b/tests/testreturn/testreturn.pro @@ -0,0 +1,10 @@ +TEMPLATE = app +TARGET = +DEPENDPATH += . +INCLUDEPATH += . + +QT -= gui +CONFIG += console + +# Input +SOURCES += main.cpp diff --git a/tests/testvm-configs/franks-host-config.cfg b/tests/testvm-configs/franks-host-config.cfg new file mode 100644 index 000000000..bddb76af9 --- /dev/null +++ b/tests/testvm-configs/franks-host-config.cfg @@ -0,0 +1,5 @@ +[General] +vmrun=vmrun +checkerInstallation=/home/frank/workspace-eclipse/test-framework/checker +testcase0=TestCases/testcase1/testcase1.cfg +vm0=VMs/NokiaTestVistaVM.cfg diff --git a/tests/testvm-configs/franks-test-vm.cfg b/tests/testvm-configs/franks-test-vm.cfg new file mode 100644 index 000000000..9bd0a113e --- /dev/null +++ b/tests/testvm-configs/franks-test-vm.cfg @@ -0,0 +1,7 @@ +[General] +snapshot=base +username=kdab +password=kdab +vmx=NokiaTestVistaVM/WindowsVista32Bit.vmx +tempDir=c:\Users\kdab\Desktop +os=windows diff --git a/tests/testvm-configs/mikes-host-config.cfg b/tests/testvm-configs/mikes-host-config.cfg new file mode 100644 index 000000000..42fb37d79 --- /dev/null +++ b/tests/testvm-configs/mikes-host-config.cfg @@ -0,0 +1,4 @@ +[General] +checkerInstallation=/Users/mike/Documents/KDAB/NokiaSDK/installer/test-framework/checker +testcase0=/Users/mike/Documents/KDAB/NokiaSDK/installer/examples/testcases/testcase1/testcase1.cfg +vm0=/Users/mike/Documents/KDAB/NokiaSDK/installer/examples/testvm-configs/mikes-test-vm.cfg \ No newline at end of file diff --git a/tests/testvm-configs/mikes-test-vm.cfg b/tests/testvm-configs/mikes-test-vm.cfg new file mode 100644 index 000000000..cc8fa4179 --- /dev/null +++ b/tests/testvm-configs/mikes-test-vm.cfg @@ -0,0 +1,7 @@ +[General] +snapshot=base +username=nokia +password=nokia +vmx=/Users/mike/Documents/Virtual Machines.localized/Nokia.vmwarevm/Nokia.vmx +tempDir=c:\Users\nokia\Desktop +os=windows \ No newline at end of file -- cgit v1.2.3