diff options
Diffstat (limited to 'util/testrunner/tests/tst_testrunner.py')
| -rwxr-xr-x | util/testrunner/tests/tst_testrunner.py | 303 |
1 files changed, 303 insertions, 0 deletions
diff --git a/util/testrunner/tests/tst_testrunner.py b/util/testrunner/tests/tst_testrunner.py new file mode 100755 index 0000000000..b2c100714d --- /dev/null +++ b/util/testrunner/tests/tst_testrunner.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + + +import sys +import os +import re +import subprocess + +from subprocess import STDOUT, PIPE +from tempfile import TemporaryDirectory, mkstemp + +MY_NAME = os.path.basename(__file__) +my_dir = os.path.dirname(__file__) +testrunner = os.path.join(my_dir, "..", "qt-testrunner.py") +mock_test = os.path.join(my_dir, "qt_mock_test.py") +xml_log_template = os.path.join(my_dir, "qt_mock_test-log.xml") + +with open(xml_log_template) as f: + XML_TEMPLATE = f.read() + + +import unittest + +def setUpModule(): + global TEMPDIR + TEMPDIR = TemporaryDirectory(prefix="tst_testrunner-") + + filename = os.path.join(TEMPDIR.name, "file_1") + print("setUpModule(): setting up temporary directory and env var" + " QT_MOCK_TEST_STATE_FILE=" + filename + " and" + " QT_MOCK_TEST_XML_TEMPLATE_FILE=" + xml_log_template) + + os.environ["QT_MOCK_TEST_STATE_FILE"] = filename + os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = xml_log_template + +def tearDownModule(): + print("\ntearDownModule(): Cleaning up temporary directory:", + TEMPDIR.name) + del os.environ["QT_MOCK_TEST_STATE_FILE"] + TEMPDIR.cleanup() + + +# Helper to run a command and always capture output +def run(*args, **kwargs): + if DEBUG: + print("Running: ", args, flush=True) + proc = subprocess.run(*args, stdout=PIPE, stderr=STDOUT, **kwargs) + if DEBUG and proc.stdout: + print(proc.stdout.decode(), flush=True) + return proc + +# Helper to run qt-testrunner.py with proper testing arguments. +def run_testrunner(xml_filename=None, extra_args=None, env=None): + + args = [ testrunner, mock_test ] + if xml_filename: + args += [ "--parse-xml-testlog", xml_filename ] + if extra_args: + args += extra_args + + return run(args, env=env) + +# Write the XML_TEMPLATE to filename, replacing the templated results. +def write_xml_log(filename, failure=None): + data = XML_TEMPLATE + # Replace what was asked to fail with "fail" + if type(failure) in (list, tuple): + for template in failure: + data = data.replace("{{"+template+"_result}}", "fail") + elif type(failure) is str: + data = data.replace("{{"+failure+"_result}}", "fail") + # Replace the rest with "pass" + data = re.sub(r"{{[^}]+}}", "pass", data) + with open(filename, "w") as f: + f.write(data) + + +# Test that qt_mock_test.py behaves well. This is necessary to properly +# test qt-testrunner. +class Test_qt_mock_test(unittest.TestCase): + def setUp(self): + state_file = os.environ["QT_MOCK_TEST_STATE_FILE"] + if os.path.exists(state_file): + os.remove(state_file) + def test_always_pass(self): + proc = run([mock_test, "always_pass"]) + self.assertEqual(proc.returncode, 0) + def test_always_fail(self): + proc = run([mock_test, "always_fail"]) + self.assertEqual(proc.returncode, 1) + def test_fail_then_pass_2(self): + proc = run([mock_test, "fail_then_pass:2"]) + self.assertEqual(proc.returncode, 1) + proc = run([mock_test, "fail_then_pass:2"]) + self.assertEqual(proc.returncode, 1) + proc = run([mock_test, "fail_then_pass:2"]) + self.assertEqual(proc.returncode, 0) + def test_fail_then_pass_1(self): + proc = run([mock_test, "fail_then_pass:1"]) + self.assertEqual(proc.returncode, 1) + proc = run([mock_test, "fail_then_pass:1"]) + self.assertEqual(proc.returncode, 0) + def test_fail_then_pass_many_tests(self): + proc = run([mock_test, "fail_then_pass:1"]) + self.assertEqual(proc.returncode, 1) + proc = run([mock_test, "fail_then_pass:2"]) + self.assertEqual(proc.returncode, 1) + proc = run([mock_test, "fail_then_pass:1"]) + self.assertEqual(proc.returncode, 0) + proc = run([mock_test, "fail_then_pass:2"]) + self.assertEqual(proc.returncode, 1) + proc = run([mock_test, "fail_then_pass:2"]) + self.assertEqual(proc.returncode, 0) + def test_xml_file_is_written(self): + filename = os.path.join(TEMPDIR.name, "testlog.xml") + proc = run([mock_test, "-o", filename+",xml"]) + self.assertEqual(proc.returncode, 0) + self.assertTrue(os.path.exists(filename)) + self.assertGreater(os.path.getsize(filename), 0) + os.remove(filename) + +# Test regular invocations of qt-testrunner. +class Test_testrunner(unittest.TestCase): + def setUp(self): + state_file = os.environ["QT_MOCK_TEST_STATE_FILE"] + if os.path.exists(state_file): + os.remove(state_file) + old_logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") + if os.path.exists(old_logfile): + os.remove(old_logfile) + self.env = dict() + self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.env["QT_MOCK_TEST_STATE_FILE"] = state_file + self.extra_args = [ "--log-dir", TEMPDIR.name ] + def prepare_env(self, run_list=None): + if run_list is not None: + self.env['QT_MOCK_TEST_RUN_LIST'] = ",".join(run_list) + def run2(self): + return run_testrunner(extra_args=self.extra_args, env=self.env) + def test_simple_invocation(self): + # All tests pass. + proc = self.run2() + self.assertEqual(proc.returncode, 0) + def test_always_pass(self): + self.prepare_env(run_list=["always_pass"]) + proc = self.run2() + self.assertEqual(proc.returncode, 0) + def test_always_fail(self): + self.prepare_env(run_list=["always_fail"]) + proc = self.run2() + # TODO verify that re-runs==max_repeats + self.assertEqual(proc.returncode, 2) + def test_flaky_pass_1(self): + self.prepare_env(run_list=["always_pass,fail_then_pass:1"]) + proc = self.run2() + self.assertEqual(proc.returncode, 0) + def test_flaky_pass_5(self): + self.prepare_env(run_list=["always_pass,fail_then_pass:1,fail_then_pass:5"]) + proc = self.run2() + self.assertEqual(proc.returncode, 0) + def test_flaky_fail(self): + self.prepare_env(run_list=["always_pass,fail_then_pass:6"]) + proc = self.run2() + self.assertEqual(proc.returncode, 2) + def test_flaky_pass_fail(self): + self.prepare_env(run_list=["always_pass,fail_then_pass:1,fail_then_pass:6"]) + proc = self.run2() + # TODO verify that one func was re-run and passed but the other failed. + self.assertEqual(proc.returncode, 2) + def test_initTestCase_fail_crash(self): + self.prepare_env(run_list=["initTestCase,always_pass"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) + + # If no XML file is found by qt-testrunner, it is usually considered a + # CRASH and the whole test is re-run. Even when the return code is zero. + # It is a PASS only if the test is not capable of XML output (see no_extra_args, TODO test it). + def test_no_xml_log_written_pass_crash(self): + del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.prepare_env(run_list=["always_pass"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) + # On the 2nd iteration of the full test, both of the tests pass. + # Still it's a CRASH because no XML file was found. + def test_no_xml_log_written_fail_then_pass_crash(self): + del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.prepare_env(run_list=["always_pass,fail_then_pass:1"]) + proc = self.run2() + # TODO verify that the whole test has run twice. + self.assertEqual(proc.returncode, 3) + # Even after 2 iterations of the full test we still get failures but no XML file, + # and this is considered a CRASH. + def test_no_xml_log_written_crash(self): + del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.prepare_env(run_list=["fail_then_pass:2"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) + + # If a test returns success but XML contains failures, it's a CRASH. + def test_wrong_xml_log_written_1_crash(self): + logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") + write_xml_log(logfile, failure="always_fail") + del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.prepare_env(run_list=["always_pass"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) + # If a test returns failure but XML contains only pass, it's a CRASH. + def test_wrong_xml_log_written_2_crash(self): + logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") + write_xml_log(logfile) + del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.prepare_env(run_list=["always_fail"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) + + +# Test qt-testrunner script with an existing XML log file: +# qt-testrunner.py qt_mock_test.py --parse-xml-testlog file.xml +# qt-testrunner should repeat the testcases that are logged as +# failures and fail or pass depending on how the testcases behave. +# Different XML files are generated for the following test cases. +# + No failure logged. qt-testrunner should exit(0) +# + The "always_pass" test has failed. qt-testrunner should exit(0). +# + The "always_fail" test has failed. qt-testrunner should exit(2). +# + The "always_crash" test has failed. qt-testrunner should exit(2). +# + The "fail_then_pass:2" test failed. qt-testrunner should exit(0). +# + The "fail_then_pass:5" test failed. qt-testrunner should exit(2). +# + The "initTestCase" failed which is listed as NO_RERUN thus +# qt-testrunner should exit(3). +class Test_testrunner_with_xml_logfile(unittest.TestCase): + # Runs before every single test function, creating a unique temp file. + def setUp(self): + (_handle, self.xml_file) = mkstemp( + suffix=".xml", prefix="qt_mock_test-log-", + dir=TEMPDIR.name) + if os.path.exists(os.environ["QT_MOCK_TEST_STATE_FILE"]): + os.remove(os.environ["QT_MOCK_TEST_STATE_FILE"]) + def tearDown(self): + os.remove(self.xml_file) + del self.xml_file + + def test_no_failure(self): + write_xml_log(self.xml_file, failure=None) + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 0) + def test_always_pass_failed(self): + write_xml_log(self.xml_file, failure="always_pass") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 0) + def test_always_pass_failed_max_repeats_0(self): + write_xml_log(self.xml_file, failure="always_pass") + proc = run_testrunner(self.xml_file, + extra_args=["--max-repeats", "0"]) + self.assertEqual(proc.returncode, 2) + def test_always_fail_failed(self): + write_xml_log(self.xml_file, failure="always_fail") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 2) + # Assert that one of the re-runs was in verbose mode + matches = re.findall("VERBOSE RUN", + proc.stdout.decode()) + self.assertEqual(len(matches), 1) + # Assert that the environment was altered too + self.assertIn("QT_LOGGING_RULES", proc.stdout.decode()) + def test_always_crash_failed(self): + write_xml_log(self.xml_file, failure="always_crash") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 2) + def test_fail_then_pass_2_failed(self): + write_xml_log(self.xml_file, failure="fail_then_pass:2") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 0) + def test_fail_then_pass_5_failed(self): + write_xml_log(self.xml_file, failure="fail_then_pass:5") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 2) + def test_with_two_failures(self): + write_xml_log(self.xml_file, + failure=["always_pass", "fail_then_pass:2"]) + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 0) + # Check that test output is properly interleaved with qt-testrunner's logging. + matches = re.findall(r"(PASS|FAIL!).*\n.*Test process exited with code", + proc.stdout.decode()) + self.assertEqual(len(matches), 4) + def test_initTestCase_fail_crash(self): + write_xml_log(self.xml_file, failure="initTestCase") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 3) + + +if __name__ == "__main__": + + DEBUG = False + if "--debug" in sys.argv: + sys.argv.remove("--debug") + DEBUG = True + + # We set failfast=True as we do not want the test suite to continue if the + # tests of qt_mock_test failed. The next ones depend on it. + unittest.main(failfast=True) |