1 files changed, 555 insertions, 0 deletions
diff --git a/tests/manual/v4/test262.py b/tests/manual/v4/test262.py
new file mode 100755
index 0000000000..c255bea1fa
--- /dev/null
+++ b/tests/manual/v4/test262.py
@@ -0,0 +1,555 @@
+#!/usr/bin/env python
+# Copyright 2009 the Sputnik authors.  All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+# This is derived from sputnik.py, the Sputnik console test runner,
+# with elements from packager.py, which is separately
+# copyrighted. TODO: Refactor so there is less duplication between
+# test262.py and packager.py.
+
+import sys
+from os import path
+rootDir = path.dirname(path.realpath(__file__))
+sys.path.insert(0, path.abspath(rootDir + "/test262/tools/packaging"))
+
+import logging
+import optparse
+import os
+import platform
+import re
+import subprocess
+import tempfile
+import time
+import xml.dom.minidom
+import datetime
+import shutil
+import json
+import stat
+import multiprocessing
+import signal
+
+
+from parseTestRecord import parseTestRecord, stripHeader
+
+from packagerConfig import *
+
+# ############# Helpers needed for parallel multi-process test execution ############
+
+def runTest(case, args):
+    return case.Run(args)
+
+def runTestVarArgs(args):
+    return runTest(*args)
+
+def initWorkerProcess():
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+# #############
+
+class Test262Error(Exception):
+  def __init__(self, message):
+    self.message = message
+
+def ReportError(s):
+  raise Test262Error(s)
+
+
+class TestExpectations:
+    def __init__(self, enabled):
+        self.testsToSkip = []
+        self.failingTests = []
+        f = open(rootDir + "/TestExpectations")
+        if not enabled:
+            return
+        for line in f.read().splitlines():
+            line = line.strip()
+            if len(line) == 0 or line[0] == "#":
+                continue
+            record = line.split()
+            if len(record) == 1:
+                self.testsToSkip.append(record[0])
+            else:
+                test = record[0]
+                expectation = record[1]
+                if expectation == "failing":
+                    self.failingTests.append(test)
+        f.close()
+
+    def update(self, progress):
+        unexpectedPasses = [c.case.name[-1] for c in progress.failed_tests if c.case.IsNegative()]
+
+        # If a test fails that we expected to fail, then it actually passed unexpectedly.
+        failures = [c.case.name[-1] for c in progress.failed_tests if not c.case.IsNegative()]
+        for failure in failures:
+            if failure in self.failingTests:
+                unexpectedPasses.append(failure)
+
+        f = open(rootDir + "/TestExpectations")
+        lines = f.read().splitlines()
+        oldLen = len(lines)
+        for result in unexpectedPasses:
+            expectationLine = result + " failing"
+            try:
+                lines.remove(expectationLine)
+            except ValueError:
+                pass
+
+        f.close()
+        if len(lines) != oldLen:
+            f = open(rootDir + "/TestExpectations", "w")
+            f.write("\n".join(lines))
+            f.close()
+            print "Changes to TestExpectations written!"
+
+
+if not os.path.exists(EXCLUDED_FILENAME):
+    print "Cannot generate (JSON) test262 tests without a file," + \
+        " %s, showing which tests have been disabled!" % EXCLUDED_FILENAME
+    sys.exit(1)
+EXCLUDE_LIST = xml.dom.minidom.parse(EXCLUDED_FILENAME)
+EXCLUDE_LIST = EXCLUDE_LIST.getElementsByTagName("test")
+EXCLUDE_LIST = [x.getAttribute("id") for x in EXCLUDE_LIST]
+
+
+def BuildOptions():
+  result = optparse.OptionParser()
+  result.add_option("--command", default="v4", help="The command-line to run")
+  result.add_option("--tests", default=path.abspath(rootDir + '/test262'),
+                    help="Path to the tests")
+  result.add_option("--cat", default=False, action="store_true",
+                    help="Print packaged test code that would be run")
+  result.add_option("--summary", default=True, action="store_true",
+                    help="Print summary after running tests")
+  result.add_option("--full-summary", default=False, action="store_true",
+                    help="Print summary and test output after running tests")
+  result.add_option("--strict_only", default=False, action="store_true", 
+                    help="Test only strict mode")
+  result.add_option("--non_strict_only", default=False, action="store_true", 
+                    help="Test only non-strict mode")
+  result.add_option("--parallel", default=False, action="store_true",
+                    help="Run tests in parallel")
+  result.add_option("--with-test-expectations", default=False, action="store_true",
+                    help="Parse TestExpectations to deal with tests known to fail")
+  result.add_option("--update-expectations", default=False, action="store_true",
+                    help="Update test expectations fail when a test passes that was expected to fail")
+  # TODO: Once enough tests are made strict compat, change the default
+  # to "both"
+  result.add_option("--unmarked_default", default="non_strict", 
+                    help="default mode for tests of unspecified strictness")
+  return result
+
+
+def ValidateOptions(options):
+  if not options.command:
+    ReportError("A --command must be specified.")
+  if not path.exists(options.tests):
+    ReportError("Couldn't find test path '%s'" % options.tests)
+
+
+placeHolderPattern = re.compile(r"\{\{(\w+)\}\}")
+
+
+def IsWindows():
+  p = platform.system()
+  return (p == 'Windows') or (p == 'Microsoft')
+
+
+class TempFile(object):
+
+  def __init__(self, suffix="", prefix="tmp", text=False):
+    self.suffix = suffix
+    self.prefix = prefix
+    self.text = text
+    self.fd = None
+    self.name = None
+    self.is_closed = False
+    self.Open()
+
+  def Open(self):
+    (self.fd, self.name) = tempfile.mkstemp(
+        suffix = self.suffix,
+        prefix = self.prefix,
+        text = self.text)
+
+  def Write(self, str):
+    os.write(self.fd, str)
+
+  def Read(self):
+    f = file(self.name)
+    result = f.read()
+    f.close()
+    return result
+
+  def Close(self):
+    if not self.is_closed:
+      self.is_closed = True
+      os.close(self.fd)
+
+  def Dispose(self):
+    try:
+      self.Close()
+      os.unlink(self.name)
+    except OSError, e:
+      logging.error("Error disposing temp file: %s", str(e))
+
+
+class TestResult(object):
+
+  def __init__(self, exit_code, stdout, stderr, case):
+    self.exit_code = exit_code
+    self.stdout = stdout
+    self.stderr = stderr
+    self.case = case
+
+  def ReportOutcome(self, long_format):
+    name = self.case.GetName()
+    mode = self.case.GetMode()
+    if self.HasUnexpectedOutcome():
+      if self.case.IsNegative():
+        print "=== %s was expected to fail in %s, but didn't ===" % (name, mode)
+      else:
+        if long_format:
+          print "=== %s failed in %s ===" % (name, mode)
+        else:
+          print "%s in %s: " % (name, mode)
+        out = self.stdout.strip()
+        if len(out) > 0:
+          print "--- output ---"
+          print out
+        err = self.stderr.strip()
+        if len(err) > 0:
+          print "--- errors ---"
+          print err
+        if long_format:
+          print "==="
+    elif self.case.IsNegative():
+      print "%s failed in %s as expected" % (name, mode)
+    else:
+      print "%s passed in %s" % (name, mode)
+
+  def HasFailed(self):
+    return self.exit_code != 0
+
+  def HasUnexpectedOutcome(self):
+    if self.case.IsNegative():
+       return not self.HasFailed()
+    else:
+       return self.HasFailed()
+
+
+class TestCase(object):
+
+  def __init__(self, suite, name, full_path, strict_mode):
+    self.suite = suite
+    self.name = name
+    self.full_path = full_path
+    self.strict_mode = strict_mode
+    f = open(self.full_path)
+    self.contents = f.read()
+    f.close()
+    testRecord = parseTestRecord(self.contents, name)
+    self.test = testRecord["test"]
+    del testRecord["test"]
+    del testRecord["header"]
+    del testRecord["commentary"]
+    self.testRecord = testRecord;
+    
+
+  def GetName(self):
+    return path.join(*self.name)
+
+  def GetMode(self):
+    if self.strict_mode:
+      return "strict mode"
+    else:
+      return "non-strict mode"
+
+  def GetPath(self):
+    return self.name
+
+  def NegateResult(self):
+      if self.IsNegative():
+          del self.testRecord['negative']
+      else:
+          self.testRecord['negative'] = "Some failure";
+
+  def IsNegative(self):
+    return 'negative' in self.testRecord
+
+  def IsOnlyStrict(self):
+    return 'onlyStrict' in self.testRecord
+
+  def IsNoStrict(self):
+    return 'noStrict' in self.testRecord
+
+  def GetSource(self):
+    # "var testDescrip = " + str(self.testRecord) + ';\n\n' + \
+    source = self.suite.GetInclude("cth.js") + \
+        self.suite.GetInclude("sta.js") + \
+        self.suite.GetInclude("ed.js") + \
+        self.suite.GetInclude("testBuiltInObject.js") + \
+        self.suite.GetInclude("testIntl.js") + \
+        self.test + '\n'
+
+    if self.strict_mode:
+      source = '"use strict";\nvar strict_mode = true;\n' + source
+    else:
+      source =  "var strict_mode = false; \n" + source
+    return source
+
+  def InstantiateTemplate(self, template, params):
+    def GetParameter(match):
+      key = match.group(1)
+      return params.get(key, match.group(0))
+    return placeHolderPattern.sub(GetParameter, template)
+
+  def Execute(self, command):
+    if IsWindows():
+      args = '%s' % command
+    else:
+      args = command.split(" ")
+    stdout = TempFile(prefix="test262-out-")
+    stderr = TempFile(prefix="test262-err-")
+    try:
+      logging.info("exec: %s", str(args))
+      process = subprocess.Popen(
+        args,
+        shell = IsWindows(),
+        stdout = stdout.fd,
+        stderr = stderr.fd
+      )
+      code = process.wait()
+      out = stdout.Read()
+      err = stderr.Read()
+    finally:
+      stdout.Dispose()
+      stderr.Dispose()
+    return (code, out, err)
+
+  def RunTestIn(self, command_template, tmp):
+    tmp.Write(self.GetSource())
+    tmp.Close()
+    command = self.InstantiateTemplate(command_template, {
+      'path': tmp.name
+    })
+    (code, out, err) = self.Execute(command)
+    return TestResult(code, out, err, self)
+
+  def Run(self, command_template):
+    tmp = TempFile(suffix=".js", prefix="test262-", text=True)
+    try:
+      result = self.RunTestIn(command_template, tmp)
+    finally:
+      tmp.Dispose()
+    return result
+
+  def Print(self):
+    print self.GetSource()
+
+
+class ProgressIndicator(object):
+
+  def __init__(self, count):
+    self.count = count
+    self.succeeded = 0
+    self.failed = 0
+    self.failed_tests = []
+
+  def HasRun(self, result):
+    result.ReportOutcome(True)
+    if result.HasUnexpectedOutcome():
+      self.failed += 1
+      self.failed_tests.append(result)
+    else:
+      self.succeeded += 1
+
+
+def MakePlural(n):
+  if (n == 1):
+    return (n, "")
+  else:
+    return (n, "s")
+
+
+class TestSuite(object):
+
+  def __init__(self, root, strict_only, non_strict_only, unmarked_default, load_expectations):
+    # TODO: derive from packagerConfig.py
+    self.test_root = path.join(root, 'test', 'suite')
+    self.lib_root = path.join(root, 'test', 'harness')
+    self.strict_only = strict_only
+    self.non_strict_only = non_strict_only
+    self.unmarked_default = unmarked_default
+    self.include_cache = { }
+    self.expectations = TestExpectations(load_expectations)
+
+  def Validate(self):
+    if not path.exists(self.test_root):
+      ReportError("No test repository found")
+    if not path.exists(self.lib_root):
+      ReportError("No test library found")
+
+  def IsHidden(self, path):
+    return path.startswith('.') or path == 'CVS'
+
+  def IsTestCase(self, path):
+    return path.endswith('.js')
+
+  def ShouldRun(self, rel_path, tests):
+    if len(tests) == 0:
+      return True
+    for test in tests:
+      if test in rel_path:
+        return True
+    return False
+
+  def GetInclude(self, name):
+    if not name in self.include_cache:
+      static = path.join(self.lib_root, name)
+      if path.exists(static):
+        f = open(static)
+        contents = stripHeader(f.read())
+        contents = re.sub(r'\r\n', '\n', contents)
+        self.include_cache[name] = contents + "\n"
+        f.close()
+      else:
+        ReportError("Can't find: " + static)
+    return self.include_cache[name]
+
+  def EnumerateTests(self, tests):
+    logging.info("Listing tests in %s", self.test_root)
+    cases = []
+    for root, dirs, files in os.walk(self.test_root):
+      for f in [x for x in dirs if self.IsHidden(x)]:
+        dirs.remove(f)
+      dirs.sort()
+      for f in sorted(files):
+        if self.IsTestCase(f):
+          full_path = path.join(root, f)
+          if full_path.startswith(self.test_root):
+            rel_path = full_path[len(self.test_root)+1:]
+          else:
+            logging.warning("Unexpected path %s", full_path)
+            rel_path = full_path
+          if self.ShouldRun(rel_path, tests) and not rel_path.startswith("intl402" + os.sep):
+            basename = path.basename(full_path)[:-3]
+            name = rel_path.split(path.sep)[:-1] + [basename]
+            if EXCLUDE_LIST.count(basename) >= 1 or self.expectations.testsToSkip.count(basename) >= 1:
+              print 'Excluded: ' + basename
+            else:
+              if not self.non_strict_only:
+                strict_case = TestCase(self, name, full_path, True)
+                if self.expectations.failingTests.count(basename) >= 1:
+                    strict_case.NegateResult()
+                if not strict_case.IsNoStrict():
+                  if strict_case.IsOnlyStrict() or \
+                        self.unmarked_default in ['both', 'strict']:
+                    cases.append(strict_case)
+              if not self.strict_only:
+                non_strict_case = TestCase(self, name, full_path, False)
+                if self.expectations.failingTests.count(basename) >= 1:
+                    non_strict_case.NegateResult()
+                if not non_strict_case.IsOnlyStrict():
+                  if non_strict_case.IsNoStrict() or \
+                        self.unmarked_default in ['both', 'non_strict']:
+                    cases.append(non_strict_case)
+    logging.info("Done listing tests")
+    return cases
+
+  def PrintSummary(self, progress):
+    print
+    print "=== Summary ==="
+    count = progress.count
+    succeeded = progress.succeeded
+    failed = progress.failed
+    print " - Ran %i test%s" % MakePlural(count)
+    if progress.failed == 0:
+      print " - All tests succeeded"
+    else:
+      percent = ((100.0 * succeeded) / count,)
+      print " - Passed %i test%s (%.1f%%)" % (MakePlural(succeeded) + percent)
+      percent = ((100.0 * failed) / count,)
+      print " - Failed %i test%s (%.1f%%)" % (MakePlural(failed) + percent)
+      positive = [c for c in progress.failed_tests if not c.case.IsNegative()]
+      negative = [c for c in progress.failed_tests if c.case.IsNegative()]
+      if len(positive) > 0:
+        print
+        print "Failed tests"
+        for result in positive:
+          print "  %s in %s" % (result.case.GetName(), result.case.GetMode())
+      if len(negative) > 0:
+        print
+        print "Expected to fail but passed ---"
+        for result in negative:
+          print " %s in %s" % (result.case.GetName(), result.case.GetMode())
+
+  def PrintFailureOutput(self, progress):
+    for result in progress.failed_tests:
+      print
+      result.ReportOutcome(False)
+
+  def Run(self, command_template, tests, print_summary, full_summary, parallel, update_expectations):
+    if not "{{path}}" in command_template:
+      command_template += " {{path}}"
+    cases = self.EnumerateTests(tests)
+    if len(cases) == 0:
+      ReportError("No tests to run")
+    progress = ProgressIndicator(len(cases))
+
+    if parallel:
+      pool = multiprocessing.Pool(processes=multiprocessing.cpu_count(), initializer=initWorkerProcess)
+      results = pool.imap_unordered(func=runTestVarArgs, iterable=[(case, command_template) for case in cases], chunksize=multiprocessing.cpu_count() * 8)
+      for result in results:
+        progress.HasRun(result)
+    else:
+      for case in cases:
+        result = case.Run(command_template)
+        progress.HasRun(result)
+    if print_summary:
+      self.PrintSummary(progress)
+      if full_summary:
+        self.PrintFailureOutput(progress)
+      else:
+        print
+        print "Use --full-summary to see output from failed tests"
+    print
+    if update_expectations:
+        self.expectations.update(progress)
+
+  def Print(self, tests):
+    cases = self.EnumerateTests(tests)
+    if len(cases) > 0:
+      cases[0].Print()
+
+
+def Main():
+  # Some date tests rely on being run in pacific time.
+  # Uncomment the next line for more logging info.
+  #logging.basicConfig(level=logging.DEBUG)
+  os.environ["TZ"] = "PST8PDT"
+  parser = BuildOptions()
+  (options, args) = parser.parse_args()
+  ValidateOptions(options)
+  test_suite = TestSuite(options.tests, 
+                         options.strict_only, 
+                         options.non_strict_only,
+                         options.unmarked_default,
+                         options.with_test_expectations)
+  test_suite.Validate()
+  if options.cat:
+    test_suite.Print(args)
+  else:
+    test_suite.Run(options.command, args,
+                   options.summary or options.full_summary,
+                   options.full_summary,
+                   options.parallel,
+                   options.update_expectations)
+
+
+if __name__ == '__main__':
+  try:
+    Main()
+    sys.exit(0)
+  except Test262Error, e:
+    print "Error: %s" % e.message
+    sys.exit(1)