Add a script to test the interpreter on *.js files.

By default this will run ./v4 on all *.js files in test262 and generate
test reports based on the exit code and the output of the interpreter.

Change-Id: I5a90b9f4c5bf376c5c8df12b5e777d0626d9d82e
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>

1 files changed, 460 insertions, 0 deletions

diff --git a/test_interpreter.py b/test_interpreter.py
new file mode 100755
index 0000000000..f69d96eb7b
--- /dev/null
+++ b/test_interpreter.py
@@ -0,0 +1,460 @@
+#!/usr/bin/env python
+
+#
+# See CommandLineProcessor.print_usage() below.
+#
+
+# TODO: Consider checking "@non_strict_only".
+
+import datetime
+import fnmatch
+import getopt
+import os
+import platform
+import resource
+import subprocess
+import threading
+import sys
+
+### Settings & Command Line Processing ########################################
+
+class Settings:
+    interpreter = 'v4'
+    interpreter_timeout = 60 # in s
+    modes = ['jit', 'interpret']
+    printers = ['txt', 'html']
+    verbose = False
+    input_dir = 'test262'
+    # List of patterns with unix shell-style wildcards, see also
+    #   http://docs.python.org/2/library/fnmatch.html
+    # Example:
+    #   blacklist_patterns = [
+    #       "test262/console/harness/cth*"
+    #   ]
+    blacklist_patterns = []
+
+class CommandLineProcessor:
+    """ Process command line arguments and overwrite values in Settings."""
+    def __init__(self, args):
+        self.args = args
+        self.valid_modes = ['aot', 'compile', 'jit', 'interpret', 'llvm-jit']
+        self.valid_printers = ['txt', 'html']
+
+    def print_usage(self):
+        sys.stdout.write("""\
+Usage: %s [-v] [-i interpreter] [-t timeout] [-d input_dir] [-p printers] [-m mode_list]
+
+Run an interpreter with different modes (mode_list) on all *.js files in
+input_dir and generate test reports based on the exit code and the output of
+the interpreter.
+
+Test reports are generated in TXT and HTML format. Paths to generated reports
+are printed upon a finished run.
+
+Options:
+  -i, --interpreter  Use the specified interpreter (default: '%s').
+  -t, --timeout      Timeout in seconds for the interpreter to process a single
+                     file (default: '%s').
+  -d, --input-dir    *.js files will be searched in input_dir. (default: '%s').
+  -m, --modes        Run the interpreter with specified modes. mode_list is a
+                     comma separated list (valid: '%s',
+                     default: '%s').
+  -p, --printers     Print results by specified printers. printers is a comma
+                     separated list (valid: '%s', default: '%s').
+  -v, --verbose      Print some more information, e.g. files are being
+                     processed (default: not enabled).
+
+""" % (
+            sys.argv[0],
+            Settings.interpreter,
+            Settings.interpreter_timeout,
+            Settings.input_dir,
+            ",".join(self.valid_modes),
+            ",".join(Settings.modes),
+            ",".join(self.valid_printers),
+            ",".join(Settings.printers),
+        ))
+
+    def run(self):
+        try:
+            options, arguments = getopt.getopt(self.args, "hvm:i:d:p:t:",
+                ["help", "verbose", "modes=", "interpreter=", "input-dir=", "printers=", "timeout="])
+        except getopt.error, msg:
+            sys.stdout.write("For help use -h, --help.")
+            sys.exit(2)
+        for option, argument in options:
+            if option in ("-h", "--help"):
+                self.print_usage()
+                sys.exit(0)
+            elif option in ("-m", "--modes"):
+                modes = filter(None, argument.split(','))
+                if not modes:
+                        sys.stdout.write("No modes provided.")
+                        sys.stdout.write("For help use -h, --help.")
+                        sys.exit(2)
+                for mode in modes:
+                    if not mode in self.valid_modes:
+                        sys.stdout.write("Mode '%s' is invalid." %(mode))
+                        sys.stdout.write("For help use -h, --help.")
+                        sys.exit(2)
+                Settings.modes = modes
+            elif option in ("-p", "--printers"):
+                printers = filter(None, argument.split(','))
+                if not printers:
+                        sys.stdout.write("No printers provided.")
+                        sys.stdout.write("For help use -h, --help.")
+                        sys.exit(2)
+                for printer in printers:
+                    if not printer in self.valid_printers:
+                        sys.stdout.write("Printer '%s' is invalid." %(printer))
+                        sys.stdout.write("For help use -h, --help.")
+                        sys.exit(2)
+                Settings.printers = printers
+            elif option in ("-i", "--interpreter"):
+                Settings.interpreter = argument
+            elif option in ("-d", "--input-dir"):
+                Settings.input_dir = argument
+            elif option in ("-t", "--interpreter-timeout"):
+                Settings.interpreter_timeout = int(argument)
+            elif option in ("-v", "--verbose"):
+                Settings.verbose = True
+            else:
+                sys.stdout.write("Unknown option '%s'." %(option))
+                sys.stdout.write("For help use -h, --help.")
+                sys.exit(2)
+
+### Utils #####################################################################
+
+def check_candidates(candidates):
+    """ Return files passing the black list Settings.blacklist_patterns. """
+    input_files = []
+    for candidate in candidates:
+        is_blacklisted = False
+        for p in Settings.blacklist_patterns:
+            if fnmatch.fnmatch(candidate, p):
+                if Settings.verbose:
+                    sys.stdout.write("Will ignore '%s' since it matches black list filter '%s'.\n" %(candidate, p))
+                is_blacklisted = True
+                break
+        if not is_blacklisted:
+            input_files.append(candidate)
+    return input_files
+
+def generic_footer(time_started, time_finished, total_pass, total_passn, total_fail, total_gout, total_time):
+    totals_processed = total_pass + total_passn + total_fail + total_gout
+    return """
+Key:
+ PASS:  Exit code is zero and _no_ output generated.
+ PASSN: Exit code is non zero and *.js contains "@negative".
+ GOUT:  Exit code is zero and output generated.
+ FAIL:  Exit code is non zero.
+ TIME:  Interpreter does not return within timeout of %d seconds.
+
+Totals: %d processed - %d passed, %d passn'ed, %d failed, %d gout'ed, %d timeout'ed.
+
+Started: %s
+Finished: %s""" %(
+        Settings.interpreter_timeout,
+        totals_processed, total_pass, total_passn, total_fail, total_gout, total_time,
+        time_started.strftime("%A, %d. %B %Y %I:%M%p"),
+        time_finished.strftime("%A, %d. %B %Y %I:%M%p")
+    )
+
+class Command(object):
+    def __init__(self, cmd):
+        self.cmd = cmd
+        self.process = None
+
+        self.finished_within_timeout = False
+        self.exit_code = None
+        self.stdout = None
+        self.stderr = None
+
+    def run(self, timeout):
+        def target():
+            my_env = os.environ.copy()
+            my_env['IN_TEST_HARNESS'] = '1'
+            self.process = subprocess.Popen(self.cmd, stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE, preexec_fn=self.__set_limits, env=my_env)
+            (self.stdout, self.stderr) = self.process.communicate()
+
+        thread = threading.Thread(target=target)
+        thread.start()
+        thread.join(timeout)
+        if thread.is_alive():
+            self.process.terminate()
+            thread.join()
+        else:
+            self.finished_within_timeout = True
+            self.exit_code = self.process.returncode
+
+    def __set_limits(self):
+        if platform.system() in ('Linux', 'Darwin'):
+            one_gigabyte = 1024*1024*1024
+            resource.setrlimit(resource.RLIMIT_AS, (one_gigabyte, one_gigabyte))
+            resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
+
+class InterpreterRunnerInfo:
+    def __init__(self, info):
+        self.info = info
+
+class InterpreterResult:
+    def __init__(self, file_path, finished_within_timeout, code, output_stdout, output_stderr):
+        self.file_path = file_path
+        self.finished_within_timeout = finished_within_timeout
+        self.code = code
+        self.output_stdout= output_stdout
+        self.output_stderr = output_stderr
+
+class InterpreterRunner:
+    def __init__(self, input_files, label, extra_args, printers):
+        self.input_files = input_files
+        self.label = label
+        self.extra_args = extra_args
+        self.printers = printers
+        self.runner_info = InterpreterRunnerInfo(Settings.interpreter + ' ' + " ".join(self.extra_args))
+
+    def __printers_begin(self):
+        for printer in self.printers:
+            printer.reset()
+            printer.set_output_file("testresults_%s.%s" %(os.path.basename(Settings.interpreter), self.label))
+            printer.set_runner_info(self.runner_info)
+            printer.print_header(datetime.datetime.now())
+
+    def __printers_end(self):
+        for printer in self.printers:
+            printer.print_footer(datetime.datetime.now())
+
+    def __printers_do(self, result):
+        for printer in self.printers:
+            printer.print_print(result)
+
+    def run(self):
+        sys.stdout.write("Running interpreter '%s %s' for each test file.\n"
+                %(Settings.interpreter, " ".join(self.extra_args)))
+        self.__printers_begin()
+        for input_file in self.input_files:
+            if not os.path.exists(input_file):
+                sys.stderr.write("Warning:  File '%s' does not exist anymore for interpreter '%s'.\n"
+                    %(input_file, self.runner_info.info))
+                break
+            if Settings.verbose:
+                sys.stdout.write("  %s\n" %(input_file))
+            command = ['./' + Settings.interpreter] + self.extra_args + [input_file]
+
+            process = Command(command)
+            process.run(timeout=Settings.interpreter_timeout)
+            if not process.finished_within_timeout:
+                sys.stderr.write("Warning: Interpreter '%s' did not finish within %d seconds processing file '%s'.\n"
+                    %(self.runner_info.info, Settings.interpreter_timeout, input_file))
+            result = InterpreterResult(
+                input_file,
+                process.finished_within_timeout,
+                process.exit_code,
+                process.stdout,
+                process.stderr
+            )
+            self.__printers_do(result)
+        self.__printers_end()
+
+### Printers ##################################################################
+
+class AbstractInterpreterPrinter:
+    def __init__(self, file_suffix):
+        self.file_suffix = file_suffix
+        self.reset()
+
+    def reset(self):
+        self.count_pass = 0
+        self.count_passn = 0
+        self.count_fail = 0
+        self.count_gout = 0
+        self.count_time = 0
+
+    def set_output_file(self, file_path):
+        self.output_file = open(file_path + self.file_suffix, 'w')
+
+    def set_runner_info(self, runner_info):
+        self.runner_info = runner_info
+
+    def print_header(self, time_started):
+        """Must be called by derived method to ensure correct totals."""
+        self.reset()
+        self.time_started = time_started
+
+    def print_print(self, result):
+        """ Returns result marker and update totals.
+            Must be called by derived method to ensure correct totals. """
+        if result.finished_within_timeout:
+            if result.code != 0:
+                if '@negative' in open(result.file_path, 'r').read(1024):
+                    result_marker = "PASSN"
+                    self.count_passn = self.count_passn + 1
+                else:
+                    result_marker = "FAIL"
+                    self.count_fail = self.count_fail + 1
+            else:
+                if result.output_stdout or result.output_stderr:
+                    result_marker = "GOUT"
+                    self.count_gout = self.count_gout + 1
+                else:
+                    result_marker = "PASS"
+                    self.count_pass = self.count_pass + 1
+        else:
+            result_marker = "TIME"
+            self.count_time = self.count_time + 1
+        return result_marker
+
+    def print_footer(self, time_finished):
+        raise NotImplementedError, "Implement me"
+
+    def generic_footer(self):
+        return generic_footer(self.time_started, self.time_finished, self.count_pass,
+            self.count_passn, self.count_fail, self.count_gout, self.count_time)
+
+class InterpreterTextPrinter(AbstractInterpreterPrinter):
+    def __init__(self):
+        AbstractInterpreterPrinter.__init__(self, '.txt')
+
+    def print_header(self, time_started):
+        AbstractInterpreterPrinter.print_header(self, time_started)
+        self.output_file.write("********* Start testing of '%s' *********\n" %(Settings.interpreter))
+        self.output_file.write("Config: %s\n" %(self.runner_info.info))
+        self.output_file.flush()
+
+    def print_print(self, result):
+        result_marker = AbstractInterpreterPrinter.print_print(self, result)
+        self.output_file.write("%-5s %-26s %s\n" %(result_marker, os.path.basename(result.file_path),
+            result.file_path))
+        self.output_file.flush()
+
+    def print_footer(self, time_finished):
+        self.time_finished = time_finished
+        self.output_file.write("********* Finished testing of '%s' *********\n" %(Settings.interpreter))
+        self.output_file.write(AbstractInterpreterPrinter.generic_footer(self))
+        self.output_file.flush()
+        sys.stdout.write("Generated '%s'.\n" %(self.output_file.name))
+
+class InterpreterHtmlPrinter(AbstractInterpreterPrinter):
+    def __init__(self):
+        AbstractInterpreterPrinter.__init__(self, '.html')
+
+    def print_header(self, time_started):
+        AbstractInterpreterPrinter.print_header(self, time_started)
+
+        preamble = ("""\
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+       "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+<title>Test results for '%s'</title>
+<style type="text/css">
+    key { font-size: smaller; }
+    table { border-collapse: collapse; }
+    body { font-family: sans-serif; }
+    th { padding: 4px; text-align: left; }
+    td { padding-left: 3px; padding-right: 3px; }
+    .pass { text-align: center; background-color:#00FF00; }
+    .passn { text-align: center; background-color:#00DE00; }
+    .fail { text-align: center; background-color:#FF4649; }
+    .gout { text-align: center; background-color:#FFDCA8; }
+    .smaller { font-size: font:1.2em/1.5em; }
+</style>
+</head>
+<body>
+<h1>Test results for '%s'</h1>
+
+<table border="1" width="100%%">
+ <tr>
+  <th style="text-align: center">Result</th>
+  <th>Test File</th>
+  <th>Interpreter Output (stdout <span style="color: red">stderr</span>)</th>
+  <th>Full Test File Path</th>
+ </tr>
+
+""" %(
+    self.runner_info.info,
+    self.runner_info.info
+    ))
+        self.output_file.write(preamble)
+        self.output_file.flush()
+
+    def print_print(self, result):
+        result_marker = AbstractInterpreterPrinter.print_print(self, result)
+        has_output = result.output_stdout or result.output_stderr
+        output = result.output_stdout + "<span style='color: red'> " + result.output_stderr + "</span>"
+        if not has_output:
+            output = ""
+        row = ("""\
+     <tr>
+      <td class="%s">%s</td>
+      <td>%s</td>
+      <td>%s</td>
+      <td>%s</td>
+     </tr>""" %(
+        result_marker.lower(),
+        result_marker,
+        os.path.basename(result.file_path),
+        output,
+        result.file_path
+        ))
+
+        self.output_file.write(row)
+        self.output_file.flush()
+
+    def print_footer(self, time_finished):
+        self.time_finished = time_finished
+        footer = ("""\
+</table>
+<pre>
+%s
+</pre>
+</body>
+</html>
+""" % (AbstractInterpreterPrinter.generic_footer(self)))
+        self.output_file.write(footer)
+        sys.stdout.write("Generated '%s'.\n" %(self.output_file.name))
+
+### Main ######################################################################
+
+def main():
+    # Handle command line options
+    cmdline_processor = CommandLineProcessor(sys.argv[1:])
+    cmdline_processor.run()
+
+    # Sanity checks
+    if not os.path.exists(Settings.interpreter):
+        sys.stderr.write("Error: Interpreter '%s' not found.\n" %(Settings.interpreter))
+        sys.exit(3)
+    if not os.path.exists(Settings.input_dir):
+        sys.stderr.write("Error: Directory with test data '%s' not found.\n" %(Settings.input_dir))
+        sys.exit(3)
+
+    # Collect files
+    candidates = []
+    for root, dirnames, filenames in os.walk(Settings.input_dir):
+      for filename in fnmatch.filter(filenames, '*.js'):
+          candidates.append(os.path.join(root, filename))
+    candidates.sort()
+    input_files = check_candidates(candidates)
+
+    if not input_files:
+        sys.stderr.write("Error: No relevant test files found in '%s'.\n" %(Settings.input_dir))
+        sys.exit(3)
+    if Settings.verbose:
+        sys.stdout.write("Will process %s files for each interpreter mode.\n" %(len(input_files)))
+
+    # Run interpreter
+    printers = []
+    for printer in Settings.printers:
+        if printer == "txt":
+            printers.append( InterpreterTextPrinter() )
+        elif printer == "html":
+            printers.append( InterpreterHtmlPrinter() )
+
+    for mode in Settings.modes:
+        runner = InterpreterRunner(input_files, mode, ["--" + mode], printers)
+        runner.run()
+
+if __name__ == "__main__":
+    main()