diff options
Diffstat (limited to 'util/testrunner/tests')
| -rw-r--r-- | util/testrunner/tests/qt_mock_test-log.xml | 36 | ||||
| -rwxr-xr-x | util/testrunner/tests/qt_mock_test.py | 182 | ||||
| -rwxr-xr-x | util/testrunner/tests/tst_testrunner.py | 303 |
3 files changed, 521 insertions, 0 deletions
diff --git a/util/testrunner/tests/qt_mock_test-log.xml b/util/testrunner/tests/qt_mock_test-log.xml new file mode 100644 index 0000000000..62e93bb8dc --- /dev/null +++ b/util/testrunner/tests/qt_mock_test-log.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="UTF-8"?> +<TestCase name="qt_mock_test"> + <Environment> + <QtVersion>MOCK</QtVersion> + <QtBuild>MOCK</QtBuild> + <QTestVersion>6.3.0</QTestVersion> + </Environment> + <TestFunction name="initTestCase"> + <Incident type="{{initTestCase_result}}" file="" line="0" /> + <Duration msecs="0.00004"/> + </TestFunction> + <TestFunction name="always_pass"> + <Incident type="{{always_pass_result}}" file="" line="0" /> + <Duration msecs="0.71704"/> + </TestFunction> + <TestFunction name="always_fail"> + <Incident type="{{always_fail_result}}" file="" line="0" /> + <Duration msecs="0.828272"/> + </TestFunction> + <TestFunction name="always_crash"> + <Incident type="{{always_crash_result}}" file="" line="0" /> + <Duration msecs="0.828272"/> + </TestFunction> + <TestFunction name="fail_then_pass"> + <Incident type="{{fail_then_pass:2_result}}" file="" line="0"> + <DataTag><![CDATA[2]]></DataTag> + </Incident> + <Incident type="{{fail_then_pass:5_result}}" file="" line="0"> + <DataTag><![CDATA[5]]></DataTag> + </Incident> + <Incident type="{{fail_then_pass:6_result}}" file="" line="0"> + <DataTag><![CDATA[6]]></DataTag> + </Incident> + </TestFunction> + <Duration msecs="1904.9"/> +</TestCase> diff --git a/util/testrunner/tests/qt_mock_test.py b/util/testrunner/tests/qt_mock_test.py new file mode 100755 index 0000000000..a7adb8804a --- /dev/null +++ b/util/testrunner/tests/qt_mock_test.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + + +# This is an artificial test, mimicking the Qt tests, for example tst_whatever. +# Its purpose is to assist in testing qt-testrunner.py. +# +# Mode A: +# +# If invoked with a test function argument, it runs that test function. +# +# Usage: +# +# $0 always_pass +# $0 always_fail +# $0 always_crash +# $0 fail_then_pass:N # where N is the number of failing runs before passing +# +# Needs environment variable: +# + QT_MOCK_TEST_STATE_FILE :: points to a unique filename, to be written +# for keeping the state of the fail_then_pass:N tests. +# +# Mode B: +# +# If invoked without any argument, it runs the tests listed in the +# variable QT_MOCK_TEST_FAIL_LIST. If variable is empty it just runs +# the always_pass test. It also understands qtestlib's `-o outfile.xml,xml` +# option for writing a mock testlog in a file. Requires environment variables: +# + QT_MOCK_TEST_STATE_FILE :: See above +# + QT_MOCK_TEST_XML_TEMPLATE_FILE :: may point to the template XML file +# located in the same source directory. Without this variable, the +# option `-o outfile.xml,xml` will be ignored. +# + QT_MOCK_TEST_FAIL_LIST :: may contain a comma-separated list of test +# that should run. + + +import sys +import os +import traceback +from tst_testrunner import write_xml_log + + +MY_NAME = os.path.basename(sys.argv[0]) +STATE_FILE = None +XML_TEMPLATE = None +XML_OUTPUT_FILE = None + + +def put_failure(test_name): + with open(STATE_FILE, "a") as f: + f.write(test_name + "\n") +def get_failures(test_name): + n = 0 + try: + with open(STATE_FILE) as f: + for line in f: + if line.strip() == test_name: + n += 1 + except FileNotFoundError: + return 0 + return n + +# Only care about the XML log output file. +def parse_output_argument(a): + global XML_OUTPUT_FILE + if a.endswith(",xml"): + XML_OUTPUT_FILE = a[:-4] + +# Strip qtestlib specific arguments. +# Only care about the "-o ...,xml" argument. +def clean_cmdline(): + args = [] + prev_arg = None + skip_next_arg = True # Skip argv[0] + for a in sys.argv: + if skip_next_arg: + if prev_arg == "-o": + parse_output_argument(a) + prev_arg = None + skip_next_arg = False + continue + if a in ("-o", "-maxwarnings"): + skip_next_arg = True + prev_arg = a + continue + if a in ("-v1", "-v2", "-vs"): + print("VERBOSE RUN") + if "QT_LOGGING_RULES" in os.environ: + print("Environment has QT_LOGGING_RULES:", + os.environ["QT_LOGGING_RULES"]) + continue + args.append(a) + return args + + +def log_test(testcase, result, + testsuite=MY_NAME.rpartition(".")[0]): + print("%-7s: %s::%s()" % (result, testsuite, testcase)) + +# Return the exit code +def run_test(testname): + if testname == "initTestCase": + exit_code = 1 # specifically test that initTestCase fails + elif testname == "always_pass": + exit_code = 0 + elif testname == "always_fail": + exit_code = 1 + elif testname == "always_crash": + exit_code = 130 + elif testname.startswith("fail_then_pass"): + wanted_fails = int(testname.partition(":")[2]) + previous_fails = get_failures(testname) + if previous_fails < wanted_fails: + put_failure(testname) + exit_code = 1 + else: + exit_code = 0 + else: + assert False, "Unknown argument: %s" % testname + + if exit_code == 0: + log_test(testname, "PASS") + elif exit_code == 1: + log_test(testname, "FAIL!") + else: + log_test(testname, "CRASH!") + + return exit_code + +def no_args_run(): + try: + run_list = os.environ["QT_MOCK_TEST_RUN_LIST"].split(",") + except KeyError: + run_list = ["always_pass"] + + total_result = True + fail_list = [] + for test in run_list: + test_exit_code = run_test(test) + if test_exit_code not in (0, 1): + sys.exit(130) # CRASH! + if test_exit_code != 0: + fail_list.append(test) + total_result = total_result and (test_exit_code == 0) + + if XML_TEMPLATE and XML_OUTPUT_FILE: + write_xml_log(XML_OUTPUT_FILE, failure=fail_list) + + if total_result: + sys.exit(0) + else: + sys.exit(1) + + +def main(): + global STATE_FILE + # Will fail if env var is not set. + STATE_FILE = os.environ["QT_MOCK_TEST_STATE_FILE"] + + global XML_TEMPLATE + if "QT_MOCK_TEST_XML_TEMPLATE_FILE" in os.environ: + with open(os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"]) as f: + XML_TEMPLATE = f.read() + + args = clean_cmdline() + + if len(args) == 0: + no_args_run() + assert False, "Unreachable!" + else: + sys.exit(run_test(args[0])) + + +# TODO write XPASS test that does exit(1) + +if __name__ == "__main__": + try: + main() + except Exception as e: + traceback.print_exc() + exit(128) # Something went wrong with this script diff --git a/util/testrunner/tests/tst_testrunner.py b/util/testrunner/tests/tst_testrunner.py new file mode 100755 index 0000000000..b2c100714d --- /dev/null +++ b/util/testrunner/tests/tst_testrunner.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + + +import sys +import os +import re +import subprocess + +from subprocess import STDOUT, PIPE +from tempfile import TemporaryDirectory, mkstemp + +MY_NAME = os.path.basename(__file__) +my_dir = os.path.dirname(__file__) +testrunner = os.path.join(my_dir, "..", "qt-testrunner.py") +mock_test = os.path.join(my_dir, "qt_mock_test.py") +xml_log_template = os.path.join(my_dir, "qt_mock_test-log.xml") + +with open(xml_log_template) as f: + XML_TEMPLATE = f.read() + + +import unittest + +def setUpModule(): + global TEMPDIR + TEMPDIR = TemporaryDirectory(prefix="tst_testrunner-") + + filename = os.path.join(TEMPDIR.name, "file_1") + print("setUpModule(): setting up temporary directory and env var" + " QT_MOCK_TEST_STATE_FILE=" + filename + " and" + " QT_MOCK_TEST_XML_TEMPLATE_FILE=" + xml_log_template) + + os.environ["QT_MOCK_TEST_STATE_FILE"] = filename + os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = xml_log_template + +def tearDownModule(): + print("\ntearDownModule(): Cleaning up temporary directory:", + TEMPDIR.name) + del os.environ["QT_MOCK_TEST_STATE_FILE"] + TEMPDIR.cleanup() + + +# Helper to run a command and always capture output +def run(*args, **kwargs): + if DEBUG: + print("Running: ", args, flush=True) + proc = subprocess.run(*args, stdout=PIPE, stderr=STDOUT, **kwargs) + if DEBUG and proc.stdout: + print(proc.stdout.decode(), flush=True) + return proc + +# Helper to run qt-testrunner.py with proper testing arguments. +def run_testrunner(xml_filename=None, extra_args=None, env=None): + + args = [ testrunner, mock_test ] + if xml_filename: + args += [ "--parse-xml-testlog", xml_filename ] + if extra_args: + args += extra_args + + return run(args, env=env) + +# Write the XML_TEMPLATE to filename, replacing the templated results. +def write_xml_log(filename, failure=None): + data = XML_TEMPLATE + # Replace what was asked to fail with "fail" + if type(failure) in (list, tuple): + for template in failure: + data = data.replace("{{"+template+"_result}}", "fail") + elif type(failure) is str: + data = data.replace("{{"+failure+"_result}}", "fail") + # Replace the rest with "pass" + data = re.sub(r"{{[^}]+}}", "pass", data) + with open(filename, "w") as f: + f.write(data) + + +# Test that qt_mock_test.py behaves well. This is necessary to properly +# test qt-testrunner. +class Test_qt_mock_test(unittest.TestCase): + def setUp(self): + state_file = os.environ["QT_MOCK_TEST_STATE_FILE"] + if os.path.exists(state_file): + os.remove(state_file) + def test_always_pass(self): + proc = run([mock_test, "always_pass"]) + self.assertEqual(proc.returncode, 0) + def test_always_fail(self): + proc = run([mock_test, "always_fail"]) + self.assertEqual(proc.returncode, 1) + def test_fail_then_pass_2(self): + proc = run([mock_test, "fail_then_pass:2"]) + self.assertEqual(proc.returncode, 1) + proc = run([mock_test, "fail_then_pass:2"]) + self.assertEqual(proc.returncode, 1) + proc = run([mock_test, "fail_then_pass:2"]) + self.assertEqual(proc.returncode, 0) + def test_fail_then_pass_1(self): + proc = run([mock_test, "fail_then_pass:1"]) + self.assertEqual(proc.returncode, 1) + proc = run([mock_test, "fail_then_pass:1"]) + self.assertEqual(proc.returncode, 0) + def test_fail_then_pass_many_tests(self): + proc = run([mock_test, "fail_then_pass:1"]) + self.assertEqual(proc.returncode, 1) + proc = run([mock_test, "fail_then_pass:2"]) + self.assertEqual(proc.returncode, 1) + proc = run([mock_test, "fail_then_pass:1"]) + self.assertEqual(proc.returncode, 0) + proc = run([mock_test, "fail_then_pass:2"]) + self.assertEqual(proc.returncode, 1) + proc = run([mock_test, "fail_then_pass:2"]) + self.assertEqual(proc.returncode, 0) + def test_xml_file_is_written(self): + filename = os.path.join(TEMPDIR.name, "testlog.xml") + proc = run([mock_test, "-o", filename+",xml"]) + self.assertEqual(proc.returncode, 0) + self.assertTrue(os.path.exists(filename)) + self.assertGreater(os.path.getsize(filename), 0) + os.remove(filename) + +# Test regular invocations of qt-testrunner. +class Test_testrunner(unittest.TestCase): + def setUp(self): + state_file = os.environ["QT_MOCK_TEST_STATE_FILE"] + if os.path.exists(state_file): + os.remove(state_file) + old_logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") + if os.path.exists(old_logfile): + os.remove(old_logfile) + self.env = dict() + self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.env["QT_MOCK_TEST_STATE_FILE"] = state_file + self.extra_args = [ "--log-dir", TEMPDIR.name ] + def prepare_env(self, run_list=None): + if run_list is not None: + self.env['QT_MOCK_TEST_RUN_LIST'] = ",".join(run_list) + def run2(self): + return run_testrunner(extra_args=self.extra_args, env=self.env) + def test_simple_invocation(self): + # All tests pass. + proc = self.run2() + self.assertEqual(proc.returncode, 0) + def test_always_pass(self): + self.prepare_env(run_list=["always_pass"]) + proc = self.run2() + self.assertEqual(proc.returncode, 0) + def test_always_fail(self): + self.prepare_env(run_list=["always_fail"]) + proc = self.run2() + # TODO verify that re-runs==max_repeats + self.assertEqual(proc.returncode, 2) + def test_flaky_pass_1(self): + self.prepare_env(run_list=["always_pass,fail_then_pass:1"]) + proc = self.run2() + self.assertEqual(proc.returncode, 0) + def test_flaky_pass_5(self): + self.prepare_env(run_list=["always_pass,fail_then_pass:1,fail_then_pass:5"]) + proc = self.run2() + self.assertEqual(proc.returncode, 0) + def test_flaky_fail(self): + self.prepare_env(run_list=["always_pass,fail_then_pass:6"]) + proc = self.run2() + self.assertEqual(proc.returncode, 2) + def test_flaky_pass_fail(self): + self.prepare_env(run_list=["always_pass,fail_then_pass:1,fail_then_pass:6"]) + proc = self.run2() + # TODO verify that one func was re-run and passed but the other failed. + self.assertEqual(proc.returncode, 2) + def test_initTestCase_fail_crash(self): + self.prepare_env(run_list=["initTestCase,always_pass"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) + + # If no XML file is found by qt-testrunner, it is usually considered a + # CRASH and the whole test is re-run. Even when the return code is zero. + # It is a PASS only if the test is not capable of XML output (see no_extra_args, TODO test it). + def test_no_xml_log_written_pass_crash(self): + del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.prepare_env(run_list=["always_pass"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) + # On the 2nd iteration of the full test, both of the tests pass. + # Still it's a CRASH because no XML file was found. + def test_no_xml_log_written_fail_then_pass_crash(self): + del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.prepare_env(run_list=["always_pass,fail_then_pass:1"]) + proc = self.run2() + # TODO verify that the whole test has run twice. + self.assertEqual(proc.returncode, 3) + # Even after 2 iterations of the full test we still get failures but no XML file, + # and this is considered a CRASH. + def test_no_xml_log_written_crash(self): + del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.prepare_env(run_list=["fail_then_pass:2"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) + + # If a test returns success but XML contains failures, it's a CRASH. + def test_wrong_xml_log_written_1_crash(self): + logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") + write_xml_log(logfile, failure="always_fail") + del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.prepare_env(run_list=["always_pass"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) + # If a test returns failure but XML contains only pass, it's a CRASH. + def test_wrong_xml_log_written_2_crash(self): + logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") + write_xml_log(logfile) + del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.prepare_env(run_list=["always_fail"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) + + +# Test qt-testrunner script with an existing XML log file: +# qt-testrunner.py qt_mock_test.py --parse-xml-testlog file.xml +# qt-testrunner should repeat the testcases that are logged as +# failures and fail or pass depending on how the testcases behave. +# Different XML files are generated for the following test cases. +# + No failure logged. qt-testrunner should exit(0) +# + The "always_pass" test has failed. qt-testrunner should exit(0). +# + The "always_fail" test has failed. qt-testrunner should exit(2). +# + The "always_crash" test has failed. qt-testrunner should exit(2). +# + The "fail_then_pass:2" test failed. qt-testrunner should exit(0). +# + The "fail_then_pass:5" test failed. qt-testrunner should exit(2). +# + The "initTestCase" failed which is listed as NO_RERUN thus +# qt-testrunner should exit(3). +class Test_testrunner_with_xml_logfile(unittest.TestCase): + # Runs before every single test function, creating a unique temp file. + def setUp(self): + (_handle, self.xml_file) = mkstemp( + suffix=".xml", prefix="qt_mock_test-log-", + dir=TEMPDIR.name) + if os.path.exists(os.environ["QT_MOCK_TEST_STATE_FILE"]): + os.remove(os.environ["QT_MOCK_TEST_STATE_FILE"]) + def tearDown(self): + os.remove(self.xml_file) + del self.xml_file + + def test_no_failure(self): + write_xml_log(self.xml_file, failure=None) + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 0) + def test_always_pass_failed(self): + write_xml_log(self.xml_file, failure="always_pass") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 0) + def test_always_pass_failed_max_repeats_0(self): + write_xml_log(self.xml_file, failure="always_pass") + proc = run_testrunner(self.xml_file, + extra_args=["--max-repeats", "0"]) + self.assertEqual(proc.returncode, 2) + def test_always_fail_failed(self): + write_xml_log(self.xml_file, failure="always_fail") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 2) + # Assert that one of the re-runs was in verbose mode + matches = re.findall("VERBOSE RUN", + proc.stdout.decode()) + self.assertEqual(len(matches), 1) + # Assert that the environment was altered too + self.assertIn("QT_LOGGING_RULES", proc.stdout.decode()) + def test_always_crash_failed(self): + write_xml_log(self.xml_file, failure="always_crash") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 2) + def test_fail_then_pass_2_failed(self): + write_xml_log(self.xml_file, failure="fail_then_pass:2") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 0) + def test_fail_then_pass_5_failed(self): + write_xml_log(self.xml_file, failure="fail_then_pass:5") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 2) + def test_with_two_failures(self): + write_xml_log(self.xml_file, + failure=["always_pass", "fail_then_pass:2"]) + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 0) + # Check that test output is properly interleaved with qt-testrunner's logging. + matches = re.findall(r"(PASS|FAIL!).*\n.*Test process exited with code", + proc.stdout.decode()) + self.assertEqual(len(matches), 4) + def test_initTestCase_fail_crash(self): + write_xml_log(self.xml_file, failure="initTestCase") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 3) + + +if __name__ == "__main__": + + DEBUG = False + if "--debug" in sys.argv: + sys.argv.remove("--debug") + DEBUG = True + + # We set failfast=True as we do not want the test suite to continue if the + # tests of qt_mock_test failed. The next ones depend on it. + unittest.main(failfast=True) |