diff options
Diffstat (limited to 'util')
| -rwxr-xr-x | util/testrunner/qt-testrunner.py | 14 | ||||
| -rwxr-xr-x | util/testrunner/tests/tst_testrunner.py | 31 |
2 files changed, 34 insertions, 11 deletions
diff --git a/util/testrunner/qt-testrunner.py b/util/testrunner/qt-testrunner.py index 8a9b469977..4dcde830ae 100755 --- a/util/testrunner/qt-testrunner.py +++ b/util/testrunner/qt-testrunner.py @@ -344,21 +344,25 @@ def main(): run_full_test(args.test_basename, args.testargs, args.log_dir, args.no_extra_args, args.dry_run, args.timeout, args.specific_extra_args) - if retcode != 0 and results_file: + if results_file: failed_functions = parse_log(results_file) if retcode == 0: + if failed_functions: + L.warning("The test executable returned success but the logfile" + f" contains FAIL for function: {failed_functions[0].func}") + continue sys.exit(0) # PASS if len(failed_functions) == 0: - L.info("No failures listed in the XML test log!" - " Did the test CRASH right after all its testcases PASSed?") + L.warning("No failures listed in the XML test log!" + " Did the test CRASH right after all its testcases PASSed?") continue cant_rerun = [ f.func for f in failed_functions if f.func in NO_RERUN_FUNCTIONS ] if cant_rerun: - L.info(f"Failure detected in the special test function '{cant_rerun[0]}'" - " which can not be re-run individually") + L.warning(f"Failure detected in the special test function '{cant_rerun[0]}'" + " which can not be re-run individually") continue assert len(failed_functions) > 0 and retcode != 0 diff --git a/util/testrunner/tests/tst_testrunner.py b/util/testrunner/tests/tst_testrunner.py index 6c16ef9612..44ba86eb65 100755 --- a/util/testrunner/tests/tst_testrunner.py +++ b/util/testrunner/tests/tst_testrunner.py @@ -202,20 +202,21 @@ class Test_testrunner(unittest.TestCase): self.assertEqual(proc.returncode, 3) # If no XML file is found by qt-testrunner, it is usually considered a - # CRASH and the whole test is re-run. But when the return code is zero, it - # doesn't care about XML file and passes anyway. - def test_no_xml_log_written_pass(self): + # CRASH and the whole test is re-run. Even when the return code is zero. + # It is a PASS only if the test is not capable of XML output (see no_extra_args, TODO test it). + def test_no_xml_log_written_pass_crash(self): del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.prepare_env(run_list=["always_pass"]) proc = self.run2() - self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.returncode, 3) # On the 2nd iteration of the full test, both of the tests pass. - def test_no_xml_log_written_fail_then_pass(self): + # Still it's a CRASH because no XML file was found. + def test_no_xml_log_written_fail_then_pass_crash(self): del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.prepare_env(run_list=["always_pass,fail_then_pass:1"]) proc = self.run2() # TODO verify that the whole test has run twice. - self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.returncode, 3) # Even after 2 iterations of the full test we still get failures but no XML file, # and this is considered a CRASH. def test_no_xml_log_written_crash(self): @@ -224,6 +225,24 @@ class Test_testrunner(unittest.TestCase): proc = self.run2() self.assertEqual(proc.returncode, 3) + # If a test returns success but XML contains failures, it's a CRASH. + def test_wrong_xml_log_written_1_crash(self): + logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") + write_xml_log(logfile, failure="always_fail") + del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.prepare_env(run_list=["always_pass"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) + # If a test returns failure but XML contains only pass, it's a CRASH. + def test_wrong_xml_log_written_2_crash(self): + logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") + write_xml_log(logfile) + del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] + self.prepare_env(run_list=["always_fail"]) + proc = self.run2() + self.assertEqual(proc.returncode, 3) + + # Test qt-testrunner script with an existing XML log file: # qt-testrunner.py qt_mock_test.py --parse-xml-testlog file.xml # qt-testrunner should repeat the testcases that are logged as |