From d4fcedddd89bbd8033c0fb1cc82c390b0a629e4f Mon Sep 17 00:00:00 2001 From: Christian Tismer Date: Sat, 25 Nov 2017 17:57:21 +0100 Subject: testrunner 1: Turn into a package Testrunner is a medium-large module that has a number of different purposes. Since we want to improve the testing logic, it is better to clean the module up before adding even more code. For compatibitity, the testrunner module will stay in place, but redirect everything into the new "testing" package. Task-number: PYSIDE-578 Change-Id: I3d14f3c4c421cdd2bb7777691b56677aa4786dd8 Reviewed-by: Friedemann Kleint --- testing/runner.py | 132 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 testing/runner.py (limited to 'testing/runner.py') diff --git a/testing/runner.py b/testing/runner.py new file mode 100644 index 000000000..b5a14e0f9 --- /dev/null +++ b/testing/runner.py @@ -0,0 +1,132 @@ +############################################################################# +## +## Copyright (C) 2017 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from __future__ import print_function + +import os +import sys +import re +import subprocess + +from collections import namedtuple + +from .buildlog import builds +from .helper import decorate, PY3, TimeoutExpired + + +class TestRunner(object): + def __init__(self, log_entry, project): + self.log_entry = log_entry + built_path = log_entry.build_dir + self.test_dir = os.path.join(built_path, project) + log_dir = log_entry.log_dir + self.logfile = os.path.join(log_dir, project + ".log") + os.environ['CTEST_OUTPUT_ON_FAILURE'] = '1' + self._setup() + + def _setup(self): + if sys.platform == 'win32': + # Windows: Helper implementing 'which' command using 'where.exe' + def winWhich(binary): + cmd = ['where.exe', binary] + stdOut = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout + result = stdOut.readlines() + stdOut.close() + if len(result) > 0: + return re.compile('\\s+').sub(' ', result[0].decode('utf-8')) + return None + + self.makeCommand = 'nmake' + qmakeSpec = os.environ.get('QMAKESPEC') + if qmakeSpec is not None and 'g++' in qmakeSpec: + self.makeCommand = 'mingw32-make' + # Can 'tee' be found in the environment (MSYS-git installation with usr/bin in path)? + self.teeCommand = winWhich('tee.exe') + if self.teeCommand is None: + git = winWhich('git.exe') + if not git: + # In COIN we have only git.cmd in path + git = winWhich('git.cmd') + if 'cmd' in git: + # Check for a MSYS-git installation with 'cmd' in the path and grab 'tee' from usr/bin + index = git.index('cmd') + self.teeCommand = git[0:index] + 'bin\\tee.exe' + if not os.path.exists(self.teeCommand): + self.teeCommand = git[0:index] + 'usr\\bin\\tee.exe' # git V2.8.X + if not os.path.exists(self.teeCommand): + raise "Cannot locate 'tee' command" + + else: + self.makeCommand = 'make' + self.teeCommand = 'tee' + + def run(self, timeout = 300): + """ + perform a test run in a given build. The build can be stopped by a + keyboard interrupt for testing this script. Also, a timeout can + be used. + """ + + if sys.platform == "win32": + cmd = (self.makeCommand, 'test') + tee_cmd = (self.teeCommand, self.logfile) + print("running", cmd, 'in', self.test_dir, ',\n logging to', self.logfile, 'using ', tee_cmd) + make = subprocess.Popen(cmd, cwd=self.test_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + tee = subprocess.Popen(tee_cmd, cwd=self.test_dir, stdin=make.stdout, shell=True) + else: + cmd = (self.makeCommand, 'test') + tee_cmd = (self.teeCommand, self.logfile) + print("running", cmd, 'in', self.test_dir, ',\n logging to', self.logfile, 'using ', tee_cmd) + make = subprocess.Popen(cmd, cwd=self.test_dir, stdout=subprocess.PIPE) + tee = subprocess.Popen(tee_cmd, cwd=self.test_dir, stdin=make.stdout) + make.stdout.close() + try: + if PY3: + output = tee.communicate(timeout=timeout)[0] + else: + output = tee.communicate()[0] + except (TimeoutExpired, KeyboardInterrupt): + print() + print("aborted") + tee.kill() + make.kill() + outs, errs = tee.communicate() + finally: + print("End of the test run") + tee.wait() -- cgit v1.2.3 From b066cee55b290261e55ec4a5b11a87c417c2a4f0 Mon Sep 17 00:00:00 2001 From: Christian Tismer Date: Sat, 25 Nov 2017 17:57:21 +0100 Subject: testrunner 3: Replace (n)make and tee by ctest Before working on the algorithm, we simplify the testrunner by using ctest directly. This has also the advantage that we can control much better how the output is generated. By interrogating the Makefile, we can now detect early if ctest was configured in setup.py . Task-number: PYSIDE-578 Change-Id: I1157576bf5806044465d7a1409c3d26810f33f62 Reviewed-by: Friedemann Kleint --- testing/runner.py | 131 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 72 insertions(+), 59 deletions(-) (limited to 'testing/runner.py') diff --git a/testing/runner.py b/testing/runner.py index b5a14e0f9..421165e56 100644 --- a/testing/runner.py +++ b/testing/runner.py @@ -45,6 +45,7 @@ import re import subprocess from collections import namedtuple +from textwrap import dedent from .buildlog import builds from .helper import decorate, PY3, TimeoutExpired @@ -60,73 +61,85 @@ class TestRunner(object): os.environ['CTEST_OUTPUT_ON_FAILURE'] = '1' self._setup() - def _setup(self): - if sys.platform == 'win32': - # Windows: Helper implementing 'which' command using 'where.exe' - def winWhich(binary): - cmd = ['where.exe', binary] - stdOut = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout - result = stdOut.readlines() - stdOut.close() - if len(result) > 0: - return re.compile('\\s+').sub(' ', result[0].decode('utf-8')) - return None + def _find_ctest(self): + """ + Find ctest in the Makefile - self.makeCommand = 'nmake' - qmakeSpec = os.environ.get('QMAKESPEC') - if qmakeSpec is not None and 'g++' in qmakeSpec: - self.makeCommand = 'mingw32-make' - # Can 'tee' be found in the environment (MSYS-git installation with usr/bin in path)? - self.teeCommand = winWhich('tee.exe') - if self.teeCommand is None: - git = winWhich('git.exe') - if not git: - # In COIN we have only git.cmd in path - git = winWhich('git.cmd') - if 'cmd' in git: - # Check for a MSYS-git installation with 'cmd' in the path and grab 'tee' from usr/bin - index = git.index('cmd') - self.teeCommand = git[0:index] + 'bin\\tee.exe' - if not os.path.exists(self.teeCommand): - self.teeCommand = git[0:index] + 'usr\\bin\\tee.exe' # git V2.8.X - if not os.path.exists(self.teeCommand): - raise "Cannot locate 'tee' command" + We no longer use make, but the ctest command directly. + It is convenient to look for the ctest program using the Makefile. + This serves us two purposes: - else: - self.makeCommand = 'make' - self.teeCommand = 'tee' + - there is no dependency of the PATH variable, + - each project is checked whether ctest was configured. + """ + make_path = os.path.join(self.test_dir, "Makefile") + look_for = "--force-new-ctest-process" + line = None + with open(make_path) as makefile: + for line in makefile: + if look_for in line: + break + else: + # We have probably forgotten to build the tests. + # Give a nice error message with a shortened but exact path. + rel_path = os.path.relpath(make_path) + msg = dedent("""\n + {line} + ** ctest is not in '{}'. + * Did you forget to build the tests with '--build-tests' in setup.py? + """).format(rel_path, line=79 * "*") + raise RuntimeError(msg) + # the ctest program is on the left to look_for + assert line, "Did not find {}".format(look_for) + ctest = re.search(r'(\S+|"([^"]+)")\s+' + look_for, line).groups() + return ctest[1] or ctest[0] + + def _setup(self): + self.ctestCommand = self._find_ctest() - def run(self, timeout = 300): + def _run(self, cmd_tuple, timeout): """ - perform a test run in a given build. The build can be stopped by a - keyboard interrupt for testing this script. Also, a timeout can - be used. + Perform a test run in a given build + + The build can be stopped by a keyboard interrupt for testing + this script. Also, a timeout can be used. + + After the change to directly using ctest, we no longer use + "--force-new-ctest-process". Until now this han no drawbacks + but was a littls faster. """ - if sys.platform == "win32": - cmd = (self.makeCommand, 'test') - tee_cmd = (self.teeCommand, self.logfile) - print("running", cmd, 'in', self.test_dir, ',\n logging to', self.logfile, 'using ', tee_cmd) - make = subprocess.Popen(cmd, cwd=self.test_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - tee = subprocess.Popen(tee_cmd, cwd=self.test_dir, stdin=make.stdout, shell=True) - else: - cmd = (self.makeCommand, 'test') - tee_cmd = (self.teeCommand, self.logfile) - print("running", cmd, 'in', self.test_dir, ',\n logging to', self.logfile, 'using ', tee_cmd) - make = subprocess.Popen(cmd, cwd=self.test_dir, stdout=subprocess.PIPE) - tee = subprocess.Popen(tee_cmd, cwd=self.test_dir, stdin=make.stdout) - make.stdout.close() + self.cmd = cmd_tuple + shell_option = sys.platform == "win32" + print(dedent("""\ + running {cmd} + in {test_dir} + """).format(**self.__dict__)) + ctest_process = subprocess.Popen(self.cmd, + cwd=self.test_dir, + stderr=subprocess.STDOUT, + shell=shell_option) try: - if PY3: - output = tee.communicate(timeout=timeout)[0] - else: - output = tee.communicate()[0] + comm = ctest_process.communicate + output = (comm(timeout=timeout) if PY3 else comm())[0] except (TimeoutExpired, KeyboardInterrupt): print() - print("aborted") - tee.kill() - make.kill() - outs, errs = tee.communicate() + print("aborted, partial resut") + ctest_process.kill() + outs, errs = ctest_process.communicate() + # ctest lists to a temp file. Move it to the log + tmp_name = self.logfile + ".tmp" + if os.path.exists(tmp_name): + if os.path.exists(self.logfile): + os.unlink(self.logfile) + os.rename(tmp_name, self.logfile) + self.partial = True + else: + self.partial = False finally: print("End of the test run") - tee.wait() + ctest_process.wait() + + def run(self, timeout=10 * 60): + cmd = self.ctestCommand, "--output-log", self.logfile + self._run(cmd, timeout) -- cgit v1.2.3 From 5c7e140036e26d8dad27ef8b7da984f093e97787 Mon Sep 17 00:00:00 2001 From: Christian Tismer Date: Thu, 7 Dec 2017 14:47:36 +0100 Subject: testrunner 5: Improve the algorithm Testrunner checks if it is running in COIN. If so, it runs each tested project 5 times and reports an error if 3 errors were found in a test function and not blacklisted. The time is measured, so we know if we can run all the tests five times or if we can run only the failed tests. At the moment, the option COIN_RERUN_FAILED_ONLY is activated by default. We can configure it by setting to false. Since this change turns the registry existence test into a flaky test (te tests generates it's missing file in the first run), the registry existence test is no more blacklisted. We simulate our own tee command now with decorated output. Task-number: PYSIDE-578 Change-Id: I6390cd50398a97a168c85c6bb778984c6a5b90fc Reviewed-by: Friedemann Kleint --- testing/runner.py | 85 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 72 insertions(+), 13 deletions(-) (limited to 'testing/runner.py') diff --git a/testing/runner.py b/testing/runner.py index 421165e56..90076f582 100644 --- a/testing/runner.py +++ b/testing/runner.py @@ -43,6 +43,7 @@ import os import sys import re import subprocess +import inspect from collections import namedtuple from textwrap import dedent @@ -52,12 +53,15 @@ from .helper import decorate, PY3, TimeoutExpired class TestRunner(object): - def __init__(self, log_entry, project): + def __init__(self, log_entry, project, index): self.log_entry = log_entry built_path = log_entry.build_dir self.test_dir = os.path.join(built_path, project) log_dir = log_entry.log_dir - self.logfile = os.path.join(log_dir, project + ".log") + if index is not None: + self.logfile = os.path.join(log_dir, project + ".{}.log".format(index)) + else: + self.logfile = os.path.join(log_dir, project + ".log") os.environ['CTEST_OUTPUT_ON_FAILURE'] = '1' self._setup() @@ -97,7 +101,7 @@ class TestRunner(object): def _setup(self): self.ctestCommand = self._find_ctest() - def _run(self, cmd_tuple, timeout): + def _run(self, cmd_tuple, label, timeout): """ Perform a test run in a given build @@ -105,26 +109,73 @@ class TestRunner(object): this script. Also, a timeout can be used. After the change to directly using ctest, we no longer use - "--force-new-ctest-process". Until now this han no drawbacks - but was a littls faster. + "--force-new-ctest-process". Until now this has no drawbacks + but was a little faster. """ self.cmd = cmd_tuple - shell_option = sys.platform == "win32" + # We no longer use the shell option. It introduces wrong handling + # of certain characters which are not yet correctly escaped: + # Especially the "^" caret char is treated as an escape, and pipe symbols + # without a caret are interpreted as such which leads to weirdness. + # Since we have all commands with explicit paths and don't use shell + # commands, this should work fine. print(dedent("""\ running {cmd} in {test_dir} """).format(**self.__dict__)) ctest_process = subprocess.Popen(self.cmd, cwd=self.test_dir, - stderr=subprocess.STDOUT, - shell=shell_option) + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + def py_tee(input, output, label): + ''' + A simple (incomplete) tee command in Python + + This script simply logs everything from input to output + while the output gets some decoration. The specific reason + to have this script at all is: + + - it is necessary to have some decoration as prefix, since + we run commands several times + + - collecting all output and then decorating is not nice if + you have to wait for a long time + + The special escape is for the case of an embedded file in + the output. + ''' + def xprint(*args, **kw): + print(*args, file=output, **kw) + + while True: + line = input.readline() + if not line: + break + labelled = True + if line.startswith('BEGIN_FILE'): + labelled = False + txt = line.rstrip() + xprint(label, txt) if label and labelled else xprint(txt) + if line.startswith('END_FILE'): + labelled = True + + tee_src = dedent("""\ + from __future__ import print_function + import sys + {} + py_tee(sys.stdin, sys.stdout, '{label}') + """).format(dedent(inspect.getsource(py_tee)), label=label) + tee_cmd = (sys.executable, "-E", "-u", "-c", tee_src) + tee_process = subprocess.Popen(tee_cmd, + cwd=self.test_dir, + stdin=ctest_process.stdout) try: - comm = ctest_process.communicate + comm = tee_process.communicate output = (comm(timeout=timeout) if PY3 else comm())[0] except (TimeoutExpired, KeyboardInterrupt): print() - print("aborted, partial resut") + print("aborted, partial result") ctest_process.kill() outs, errs = ctest_process.communicate() # ctest lists to a temp file. Move it to the log @@ -138,8 +189,16 @@ class TestRunner(object): self.partial = False finally: print("End of the test run") - ctest_process.wait() + print() + tee_process.wait() - def run(self, timeout=10 * 60): + def run(self, label, rerun, timeout): cmd = self.ctestCommand, "--output-log", self.logfile - self._run(cmd, timeout) + if rerun is not None: + # cmd += ("--rerun-failed",) + # For some reason, this worked never in the script file. + # We pass instead the test names as a regex: + words = "^(" + "|".join(rerun) + ")$" + cmd += ("--tests-regex", words) + self._run(cmd, label, timeout) +# eof -- cgit v1.2.3 From a18994c2ff7836a5b6998f6aee24583d920dad80 Mon Sep 17 00:00:00 2001 From: Christian Tismer Date: Thu, 21 Dec 2017 19:44:25 +0100 Subject: small enhancement for testrunner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new testrunner now has the multiple test feature. We refine and correct the summary view a little bit, remove more dead code, add some comments and improve the output slightly. Change-Id: I152bf5cbe2171fb07de7e88054a42fc767dc14c0 Reviewed-by: Friedemann Kleint Reviewed-by: Qt CI Bot Reviewed-by: Simo Fält --- testing/runner.py | 1 + 1 file changed, 1 insertion(+) (limited to 'testing/runner.py') diff --git a/testing/runner.py b/testing/runner.py index 90076f582..a993c3a5c 100644 --- a/testing/runner.py +++ b/testing/runner.py @@ -148,6 +148,7 @@ class TestRunner(object): def xprint(*args, **kw): print(*args, file=output, **kw) + # 'for line in input:' would read into too large chunks while True: line = input.readline() if not line: -- cgit v1.2.3