diff options
author | Dimitrios Apostolou <jimis@qt.io> | 2022-01-27 20:12:10 +0100 |
---|---|---|
committer | Dimitrios Apostolou <jimis@qt.io> | 2022-01-30 01:00:35 +0100 |
commit | b4d9d5c89c82e47b03de22ba896ad6c0474bcf5a (patch) | |
tree | fe6d207951dadc23d1be0b607e40195d15dd9a05 /util/testrunner | |
parent | 1ae5b3628d0858221542993331ede4c3ee7b3630 (diff) |
qt-testrunner: do not try to re-run initTestCase and others
Some function names are special for qtestlib, in the sense that they can
not be specified as a command line argument to run individually.
In such cases qt-testrunner treats the failure specially and tries once
to re-run the full test executable.
Fixes: QTBUG-89011
Change-Id: I0cc25f91c57374e5ac65ade10e2e223fe969f211
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>
Reviewed-by: Daniel Smith <Daniel.Smith@qt.io>
Diffstat (limited to 'util/testrunner')
-rwxr-xr-x | util/testrunner/qt-testrunner.py | 58 | ||||
-rw-r--r-- | util/testrunner/tests/qt_mock_test-log.xml | 4 | ||||
-rwxr-xr-x | util/testrunner/tests/qt_mock_test.py | 4 | ||||
-rwxr-xr-x | util/testrunner/tests/tst_testrunner.py | 10 |
4 files changed, 53 insertions, 23 deletions
diff --git a/util/testrunner/qt-testrunner.py b/util/testrunner/qt-testrunner.py index 0020bd5fdf..d0d54f054c 100755 --- a/util/testrunner/qt-testrunner.py +++ b/util/testrunner/qt-testrunner.py @@ -64,9 +64,9 @@ # 0: PASS. Either no test failed, or failed initially but passed # in the re-runs (FLAKY PASS). # 1: Some unexpected error of this script. -# 2: FAIL! for at least one test, even after the re-runs. +# 2: FAIL! for at least one test, even after the individual re-runs. # 3: CRASH! for the test executable even after re-running it once. - +# Or when we can't re-run individual functions for any reason. import sys @@ -99,6 +99,10 @@ VERBOSE_ENV = { "QT_LOGGING_RULES": "*=true", "QT_MESSAGE_PATTERN": "[%{time process} %{if-debug}D%{endif}%{if-warning}W%{endif}%{if-critical}C%{endif}%{if-fatal}F%{endif}] %{category} %{file}:%{line} %{function}() - %{message}", } +# The following special function names can not re-run individually. +NO_RERUN_FUNCTIONS = { + "initTestCase", "init", "cleanup", "cleanupTestCase" +} def parse_args(): @@ -300,39 +304,49 @@ def main(): args = parse_args() n_full_runs = 1 if args.parse_xml_testlog else 2 - for i in range(n_full_runs): + for i in range(n_full_runs + 1): + + if 0 < i < n_full_runs: + L.info("Will re-run the full test executable") + elif i == n_full_runs: # Failed on the final run + L.error("Full test run failed repeatedly, aborting!") + sys.exit(3) + try: - if i != 0: - L.info("Re-running the full test!") - if args.parse_xml_testlog: - retcode = 1 # pretend the test returned error - results_file = args.parse_xml_testlog - else: + failed_functions = [] + if args.parse_xml_testlog: # do not run test, just parse file + failed_functions = parse_log(args.parse_xml_testlog) + # Pretend the test returned correct exit code + retcode = len(failed_functions) + else: # normal invocation, run test (retcode, results_file) = \ run_full_test(args.test_basename, args.testargs, args.log_dir, args.no_extra_args, args.dry_run, args.timeout, args.specific_extra_args) - if retcode == 0: - sys.exit(0) # PASS + if retcode != 0 and results_file: + failed_functions = parse_log(results_file) + + if retcode == 0: + sys.exit(0) # PASS - failed_functions = parse_log(results_file) + if len(failed_functions) == 0: + L.info("No failures listed in the XML test log!" + " Did the test CRASH right after all its testcases PASSed?") + continue - if not args.parse_xml_testlog: - assert len(failed_functions) > 0, \ - "The XML test log should contain at least one failure!" \ - " Did the test CRASH right after all its testcases PASSed?" + cant_rerun = [ f.func for f in failed_functions if f.func in NO_RERUN_FUNCTIONS ] + if cant_rerun: + L.info(f"Failure detected in the special test function '{cant_rerun[0]}'" + " which can not be re-run individually") + continue - break # go to re-running individual failed testcases + assert len(failed_functions) > 0 and retcode != 0 + break # all is fine, goto re-running individual failed testcases except Exception as e: L.exception("The test executable CRASHed uncontrollably!" " Details about where we caught the problem:", exc_info=e) - if i < n_full_runs - 1: - L.info("Will re-run the full test executable") - else: # Failed on the final run - L.error("Full test run failed repeatedly, aborting!") - sys.exit(3) if args.max_repeats == 0: sys.exit(2) # Some tests failed but no re-runs were asked diff --git a/util/testrunner/tests/qt_mock_test-log.xml b/util/testrunner/tests/qt_mock_test-log.xml index 0c316d71c3..62e93bb8dc 100644 --- a/util/testrunner/tests/qt_mock_test-log.xml +++ b/util/testrunner/tests/qt_mock_test-log.xml @@ -5,6 +5,10 @@ <QtBuild>MOCK</QtBuild> <QTestVersion>6.3.0</QTestVersion> </Environment> + <TestFunction name="initTestCase"> + <Incident type="{{initTestCase_result}}" file="" line="0" /> + <Duration msecs="0.00004"/> + </TestFunction> <TestFunction name="always_pass"> <Incident type="{{always_pass_result}}" file="" line="0" /> <Duration msecs="0.71704"/> diff --git a/util/testrunner/tests/qt_mock_test.py b/util/testrunner/tests/qt_mock_test.py index 23d8758190..d4e6eca140 100755 --- a/util/testrunner/tests/qt_mock_test.py +++ b/util/testrunner/tests/qt_mock_test.py @@ -127,7 +127,9 @@ def log_test(testcase, result, # Return the exit code def run_test(testname): - if testname == "always_pass": + if testname == "initTestCase": + exit_code = 1 # specifically test that initTestCase fails + elif testname == "always_pass": exit_code = 0 elif testname == "always_fail": exit_code = 1 diff --git a/util/testrunner/tests/tst_testrunner.py b/util/testrunner/tests/tst_testrunner.py index e826ccb305..6c16ef9612 100755 --- a/util/testrunner/tests/tst_testrunner.py +++ b/util/testrunner/tests/tst_testrunner.py @@ -196,6 +196,10 @@ class Test_testrunner(unittest.TestCase): proc = self.run2() # TODO verify that one func was re-run and passed but the other failed. self.assertEqual(proc.returncode, 2) + def test_initTestCase_fail_crash(self): + self.prepare_env(run_list=["initTestCase,always_pass"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) # If no XML file is found by qt-testrunner, it is usually considered a # CRASH and the whole test is re-run. But when the return code is zero, it @@ -231,6 +235,8 @@ class Test_testrunner(unittest.TestCase): # + The "always_crash" test has failed. qt-testrunner should exit(2). # + The "fail_then_pass:2" test failed. qt-testrunner should exit(0). # + The "fail_then_pass:5" test failed. qt-testrunner should exit(2). +# + The "initTestCase" failed which is listed as NO_RERUN thus +# qt-testrunner should exit(3). class Test_testrunner_with_xml_logfile(unittest.TestCase): # Runs before every single test function, creating a unique temp file. def setUp(self): @@ -287,6 +293,10 @@ class Test_testrunner_with_xml_logfile(unittest.TestCase): matches = re.findall(r"(PASS|FAIL!).*\n.*Test process exited with code", proc.stdout.decode()) self.assertEqual(len(matches), 4) + def test_initTestCase_fail_crash(self): + write_xml_log(self.xml_file, failure="initTestCase") + proc = run_testrunner(self.xml_file) + self.assertEqual(proc.returncode, 3) if __name__ == "__main__": |