Merge remote-tracking branch 'origin/5.6' into 5.9

Change-Id: I254cc725505349e73c7abc290afc7abb85156997

1 files changed, 219 insertions, 0 deletions

diff --git a/testing/runner.py b/testing/runner.py
new file mode 100644
index 000000000..f86bdeebf
--- /dev/null
+++ b/testing/runner.py
@@ -0,0 +1,219 @@
+#############################################################################
+##
+## Copyright (C) 2017 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of PySide2.
+##
+## $QT_BEGIN_LICENSE:LGPL$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 3 as published by the Free Software
+## Foundation and appearing in the file LICENSE.LGPL3 included in the
+## packaging of this file. Please review the following information to
+## ensure the GNU Lesser General Public License version 3 requirements
+## will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 2.0 or (at your option) the GNU General
+## Public license version 3 or any later version approved by the KDE Free
+## Qt Foundation. The licenses are as published by the Free Software
+## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-2.0.html and
+## https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+from __future__ import print_function
+
+import os
+import sys
+import re
+import subprocess
+import inspect
+
+from collections import namedtuple
+from textwrap import dedent
+
+from .buildlog import builds
+from .helper import decorate, PY3, TimeoutExpired
+
+sys.path.append('..')
+from utils import detectClang
+
+class TestRunner(object):
+    def __init__(self, log_entry, project, index):
+        self.log_entry = log_entry
+        built_path = log_entry.build_dir
+        self.test_dir = os.path.join(built_path, project)
+        log_dir = log_entry.log_dir
+        if index is not None:
+            self.logfile = os.path.join(log_dir, project + ".{}.log".format(index))
+        else:
+            self.logfile = os.path.join(log_dir, project + ".log")
+        os.environ['CTEST_OUTPUT_ON_FAILURE'] = '1'
+        self._setupClang()
+        self._setup()
+
+    def _setupClang(self):
+        if sys.platform != "win32":
+            return
+        clangDir = detectClang()
+        if clangDir[0]:
+            clangBinDir = os.path.join(clangDir[0], 'bin')
+            path = os.environ.get('PATH')
+            if not clangBinDir in path:
+                os.environ['PATH'] = clangBinDir + os.pathsep + path
+                print("Adding %s as detected by %s to PATH" % (clangBinDir, clangDir[1]))
+
+    def _find_ctest(self):
+        """
+        Find ctest in the Makefile
+
+        We no longer use make, but the ctest command directly.
+        It is convenient to look for the ctest program using the Makefile.
+        This serves us two purposes:
+
+          - there is no dependency of the PATH variable,
+          - each project is checked whether ctest was configured.
+        """
+        make_path = os.path.join(self.test_dir, "Makefile")
+        look_for = "--force-new-ctest-process"
+        line = None
+        with open(make_path) as makefile:
+            for line in makefile:
+                if look_for in line:
+                    break
+            else:
+                # We have probably forgotten to build the tests.
+                # Give a nice error message with a shortened but exact path.
+                rel_path = os.path.relpath(make_path)
+                msg = dedent("""\n
+                    {line}
+                    **  ctest is not in '{}'.
+                    *   Did you forget to build the tests with '--build-tests' in setup.py?
+                    """).format(rel_path, line=79 * "*")
+                raise RuntimeError(msg)
+        # the ctest program is on the left to look_for
+        assert line, "Did not find {}".format(look_for)
+        ctest = re.search(r'(\S+|"([^"]+)")\s+' + look_for, line).groups()
+        return ctest[1] or ctest[0]
+
+    def _setup(self):
+        self.ctestCommand = self._find_ctest()
+
+    def _run(self, cmd_tuple, label, timeout):
+        """
+        Perform a test run in a given build
+
+        The build can be stopped by a keyboard interrupt for testing
+        this script. Also, a timeout can be used.
+
+        After the change to directly using ctest, we no longer use
+        "--force-new-ctest-process". Until now this has no drawbacks
+        but was a little faster.
+        """
+
+        self.cmd = cmd_tuple
+        # We no longer use the shell option. It introduces wrong handling
+        # of certain characters which are not yet correctly escaped:
+        # Especially the "^" caret char is treated as an escape, and pipe symbols
+        # without a caret are interpreted as such which leads to weirdness.
+        # Since we have all commands with explicit paths and don't use shell
+        # commands, this should work fine.
+        print(dedent("""\
+            running {cmd}
+                 in {test_dir}
+            """).format(**self.__dict__))
+        ctest_process = subprocess.Popen(self.cmd,
+                                         cwd=self.test_dir,
+                                         stdout=subprocess.PIPE,
+                                         stderr=subprocess.STDOUT)
+        def py_tee(input, output, label):
+            '''
+            A simple (incomplete) tee command in Python
+
+            This script simply logs everything from input to output
+            while the output gets some decoration. The specific reason
+            to have this script at all is:
+
+            - it is necessary to have some decoration as prefix, since
+              we run commands several times
+
+            - collecting all output and then decorating is not nice if
+              you have to wait for a long time
+
+            The special escape is for the case of an embedded file in
+            the output.
+            '''
+            def xprint(*args, **kw):
+                print(*args, file=output, **kw)
+
+            # 'for line in input:' would read into too large chunks
+            while True:
+                line = input.readline()
+                if not line:
+                    break
+                labelled = True
+                if line.startswith('BEGIN_FILE'):
+                    labelled = False
+                txt = line.rstrip()
+                xprint(label, txt) if label and labelled else xprint(txt)
+                if line.startswith('END_FILE'):
+                    labelled = True
+
+        tee_src = dedent("""\
+            from __future__ import print_function
+            import sys
+            {}
+            py_tee(sys.stdin, sys.stdout, '{label}')
+            """).format(dedent(inspect.getsource(py_tee)), label=label)
+        tee_cmd = (sys.executable, "-E", "-u", "-c", tee_src)
+        tee_process = subprocess.Popen(tee_cmd,
+                                       cwd=self.test_dir,
+                                       stdin=ctest_process.stdout)
+        try:
+            comm = tee_process.communicate
+            output = (comm(timeout=timeout) if PY3 else comm())[0]
+        except (TimeoutExpired, KeyboardInterrupt):
+            print()
+            print("aborted, partial result")
+            ctest_process.kill()
+            outs, errs = ctest_process.communicate()
+            # ctest lists to a temp file. Move it to the log
+            tmp_name = self.logfile + ".tmp"
+            if os.path.exists(tmp_name):
+                if os.path.exists(self.logfile):
+                    os.unlink(self.logfile)
+                os.rename(tmp_name, self.logfile)
+            self.partial = True
+        else:
+            self.partial = False
+        finally:
+            print("End of the test run")
+            print()
+        tee_process.wait()
+
+    def run(self, label, rerun, timeout):
+        cmd = self.ctestCommand, "--output-log", self.logfile
+        if rerun is not None:
+            # cmd += ("--rerun-failed",)
+            # For some reason, this worked never in the script file.
+            # We pass instead the test names as a regex:
+            words = "^(" + "|".join(rerun) + ")$"
+            cmd += ("--tests-regex", words)
+        self._run(cmd, label, timeout)
+# eof