diff options
Diffstat (limited to 'chromium/third_party/catapult/common/py_vulcanize/py_vulcanize')
22 files changed, 2944 insertions, 0 deletions
diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/__init__.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/__init__.py new file mode 100644 index 00000000000..f3a4bd1bb15 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Trace-viewer component model. + +This module implements trace-viewer's component model. +""" + +from py_vulcanize.generate import * # pylint: disable=wildcard-import +from py_vulcanize.project import Project diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/fake_fs.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/fake_fs.py new file mode 100644 index 00000000000..dfcb5e60bf4 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/fake_fs.py @@ -0,0 +1,146 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import codecs +import os +import sys +import collections +import StringIO + + +class WithableStringIO(StringIO.StringIO): + + def __enter__(self, *args): + return self + + def __exit__(self, *args): + pass + + +class FakeFS(object): + + def __init__(self, initial_filenames_and_contents=None): + self._file_contents = {} + if initial_filenames_and_contents: + for k, v in initial_filenames_and_contents.iteritems(): + self._file_contents[k] = v + + self._bound = False + self._real_codecs_open = codecs.open + self._real_open = sys.modules['__builtin__'].open + self._real_abspath = os.path.abspath + self._real_exists = os.path.exists + self._real_walk = os.walk + self._real_listdir = os.listdir + + def __enter__(self): + self.Bind() + return self + + def __exit__(self, *args): + self.Unbind() + + def Bind(self): + assert not self._bound + codecs.open = self._FakeCodecsOpen + sys.modules['__builtin__'].open = self._FakeOpen + os.path.abspath = self._FakeAbspath + os.path.exists = self._FakeExists + os.walk = self._FakeWalk + os.listdir = self._FakeListDir + self._bound = True + + def Unbind(self): + assert self._bound + codecs.open = self._real_codecs_open + sys.modules['__builtin__'].open = self._real_open + os.path.abspath = self._real_abspath + os.path.exists = self._real_exists + os.walk = self._real_walk + os.listdir = self._real_listdir + self._bound = False + + def AddFile(self, path, contents): + assert path not in self._file_contents + path = os.path.normpath(path) + self._file_contents[path] = contents + + def _FakeOpen(self, path, mode=None): + if mode is None: + mode = 'r' + if mode == 'r' or mode == 'rU' or mode == 'rb': + if path not in self._file_contents: + return self._real_open(path, mode) + return WithableStringIO(self._file_contents[path]) + + raise NotImplementedError() + + def _FakeCodecsOpen(self, path, mode=None, + encoding=None): # pylint: disable=unused-argument + if mode is None: + mode = 'r' + if mode == 'r' or mode == 'rU' or mode == 'rb': + if path not in self._file_contents: + return self._real_open(path, mode) + return WithableStringIO(self._file_contents[path]) + + raise NotImplementedError() + + def _FakeAbspath(self, path): + """Normalize the path and ensure it starts with os.path.sep. + + The tests all assume paths start with things like '/my/project', + and this abspath implementaion makes that assumption work correctly + on Windows. + """ + normpath = os.path.normpath(path) + if not normpath.startswith(os.path.sep): + normpath = os.path.sep + normpath + return normpath + + def _FakeExists(self, path): + if path in self._file_contents: + return True + return self._real_exists(path) + + def _FakeWalk(self, top): + assert os.path.isabs(top) + all_filenames = self._file_contents.keys() + pending_prefixes = collections.deque() + pending_prefixes.append(top) + visited_prefixes = set() + while len(pending_prefixes): + prefix = pending_prefixes.popleft() + if prefix in visited_prefixes: + continue + visited_prefixes.add(prefix) + if prefix.endswith(os.path.sep): + prefix_with_trailing_sep = prefix + else: + prefix_with_trailing_sep = prefix + os.path.sep + + dirs = set() + files = [] + for filename in all_filenames: + if not filename.startswith(prefix_with_trailing_sep): + continue + relative_to_prefix = os.path.relpath(filename, prefix) + + dirpart = os.path.dirname(relative_to_prefix) + if len(dirpart) == 0: + files.append(relative_to_prefix) + continue + parts = dirpart.split(os.sep) + if len(parts) == 0: + dirs.add(dirpart) + else: + pending = os.path.join(prefix, parts[0]) + dirs.add(parts[0]) + pending_prefixes.appendleft(pending) + + dirs = sorted(dirs) + yield prefix, dirs, files + + def _FakeListDir(self, dirname): + raise NotImplementedError() diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/fake_fs_unittest.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/fake_fs_unittest.py new file mode 100644 index 00000000000..08250138a5e --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/fake_fs_unittest.py @@ -0,0 +1,52 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import unittest + +from py_vulcanize import fake_fs + + +class FakeFSUnittest(unittest.TestCase): + + def testBasic(self): + fs = fake_fs.FakeFS() + fs.AddFile('/blah/x', 'foobar') + with fs: + assert os.path.exists(os.path.normpath('/blah/x')) + self.assertEquals( + 'foobar', + open(os.path.normpath('/blah/x'), 'r').read()) + + def testWithableOpen(self): + fs = fake_fs.FakeFS() + fs.AddFile('/blah/x', 'foobar') + with fs: + with open(os.path.normpath('/blah/x'), 'r') as f: + self.assertEquals('foobar', f.read()) + + def testWalk(self): + fs = fake_fs.FakeFS() + fs.AddFile('/x/w2/w3/z3.txt', '') + fs.AddFile('/x/w/z.txt', '') + fs.AddFile('/x/y.txt', '') + fs.AddFile('/a.txt', 'foobar') + with fs: + gen = os.walk(os.path.normpath('/')) + r = gen.next() + self.assertEquals((os.path.normpath('/'), ['x'], ['a.txt']), r) + + r = gen.next() + self.assertEquals((os.path.normpath('/x'), ['w', 'w2'], ['y.txt']), r) + + r = gen.next() + self.assertEquals((os.path.normpath('/x/w'), [], ['z.txt']), r) + + r = gen.next() + self.assertEquals((os.path.normpath('/x/w2'), ['w3'], []), r) + + r = gen.next() + self.assertEquals((os.path.normpath('/x/w2/w3'), [], ['z3.txt']), r) + + self.assertRaises(StopIteration, gen.next) diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/generate.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/generate.py new file mode 100644 index 00000000000..f6accb4b328 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/generate.py @@ -0,0 +1,274 @@ +# Copyright (c) 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import sys +import subprocess +import tempfile + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +from py_vulcanize import html_generation_controller + + +html_warning_message = """ + + +<!-- +WARNING: This file is auto generated. + + Do not edit directly. +--> +""" + +js_warning_message = """ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/* WARNING: This file is auto generated. + * + * Do not edit directly. + */ +""" + +css_warning_message = """ +/* Copyright 2015 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. */ + +/* WARNING: This file is auto-generated. + * + * Do not edit directly. + */ +""" + + +def _AssertIsUTF8(f): + if isinstance(f, StringIO): + return + assert f.encoding == 'utf-8' + + +def _MinifyJS(input_js): + py_vulcanize_path = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..')) + rjsmin_path = os.path.abspath( + os.path.join(py_vulcanize_path, 'third_party', 'rjsmin', 'rjsmin.py')) + + with tempfile.NamedTemporaryFile() as _: + args = [ + 'python', + rjsmin_path + ] + p = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + res = p.communicate(input=input_js) + errorcode = p.wait() + if errorcode != 0: + sys.stderr.write('rJSmin exited with error code %d' % errorcode) + sys.stderr.write(res[1]) + raise Exception('Failed to minify, omgah') + return res[0] + + +def GenerateJS(load_sequence, + use_include_tags_for_scripts=False, + dir_for_include_tag_root=None, + minify=False, + report_sizes=False): + f = StringIO() + GenerateJSToFile(f, + load_sequence, + use_include_tags_for_scripts, + dir_for_include_tag_root, + minify=minify, + report_sizes=report_sizes) + + return f.getvalue() + + +def GenerateJSToFile(f, + load_sequence, + use_include_tags_for_scripts=False, + dir_for_include_tag_root=None, + minify=False, + report_sizes=False): + _AssertIsUTF8(f) + if use_include_tags_for_scripts and dir_for_include_tag_root is None: + raise Exception('Must provide dir_for_include_tag_root') + + f.write(js_warning_message) + f.write('\n') + + if not minify: + flatten_to_file = f + else: + flatten_to_file = StringIO() + + for module in load_sequence: + module.AppendJSContentsToFile(flatten_to_file, + use_include_tags_for_scripts, + dir_for_include_tag_root) + if minify: + js = flatten_to_file.getvalue() + minified_js = _MinifyJS(js) + f.write(minified_js) + f.write('\n') + + if report_sizes: + for module in load_sequence: + s = StringIO() + module.AppendJSContentsToFile(s, + use_include_tags_for_scripts, + dir_for_include_tag_root) + + # Add minified size info. + js = s.getvalue() + min_js_size = str(len(_MinifyJS(js))) + + # Print names for this module. Some domain-specific simplifications + # are included to make pivoting more obvious. + parts = module.name.split('.') + if parts[:2] == ['base', 'ui']: + parts = ['base_ui'] + parts[2:] + if parts[:2] == ['tracing', 'importer']: + parts = ['importer'] + parts[2:] + tln = parts[0] + sln = '.'.join(parts[:2]) + + # Output + print('%i\t%s\t%s\t%s\t%s' % ( + len(js), min_js_size, module.name, tln, sln)) + sys.stdout.flush() + + +class ExtraScript(object): + + def __init__(self, script_id=None, text_content=None, content_type=None): + if script_id is not None: + assert script_id[0] != '#' + self.script_id = script_id + self.text_content = text_content + self.content_type = content_type + + def WriteToFile(self, output_file): + _AssertIsUTF8(output_file) + attrs = [] + if self.script_id: + attrs.append('id="%s"' % self.script_id) + if self.content_type: + attrs.append('content-type="%s"' % self.content_type) + + if len(attrs) > 0: + output_file.write('<script %s>\n' % ' '.join(attrs)) + else: + output_file.write('<script>\n') + if self.text_content: + output_file.write(self.text_content) + output_file.write('</script>\n') + + +def _MinifyCSS(css_text): + py_vulcanize_path = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..')) + rcssmin_path = os.path.abspath( + os.path.join(py_vulcanize_path, 'third_party', 'rcssmin', 'rcssmin.py')) + + with tempfile.NamedTemporaryFile() as _: + rcssmin_args = ['python', rcssmin_path] + p = subprocess.Popen(rcssmin_args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + res = p.communicate(input=css_text) + errorcode = p.wait() + if errorcode != 0: + sys.stderr.write('rCSSmin exited with error code %d' % errorcode) + sys.stderr.write(res[1]) + raise Exception('Failed to generate css for %s.' % css_text) + return res[0] + + +def GenerateStandaloneHTMLAsString(*args, **kwargs): + f = StringIO() + GenerateStandaloneHTMLToFile(f, *args, **kwargs) + return f.getvalue() + + +def GenerateStandaloneHTMLToFile(output_file, + load_sequence, + title=None, + flattened_js_url=None, + extra_scripts=None, + minify=False, + report_sizes=False, + output_html_head_and_body=True): + """Writes a HTML file with the content of all modules in a load sequence. + + The load_sequence is a list of (HTML or JS) Module objects; the order that + they're inserted into the file depends on their type and position in the load + sequence. + """ + _AssertIsUTF8(output_file) + extra_scripts = extra_scripts or [] + + if output_html_head_and_body: + output_file.write( + '<!DOCTYPE html>\n' + '<html>\n' + ' <head i18n-values="dir:textdirection;">\n' + ' <meta http-equiv="Content-Type" content="text/html;' + 'charset=utf-8">\n') + if title: + output_file.write(' <title>%s</title>\n ' % title) + else: + assert title is None + + loader = load_sequence[0].loader + + written_style_sheets = set() + + class HTMLGenerationController( + html_generation_controller.HTMLGenerationController): + + def __init__(self, module): + self.module = module + + def GetHTMLForStylesheetHRef(self, href): + resource = self.module.HRefToResource( + href, '<link rel="stylesheet" href="%s">' % href) + style_sheet = loader.LoadStyleSheet(resource.name) + + if style_sheet in written_style_sheets: + return None + written_style_sheets.add(style_sheet) + + text = style_sheet.contents_with_inlined_images + if minify: + text = _MinifyCSS(text) + return '<style>\n%s\n</style>' % text + + for module in load_sequence: + controller = HTMLGenerationController(module) + module.AppendHTMLContentsToFile(output_file, controller, minify=minify) + + if flattened_js_url: + output_file.write('<script src="%s"></script>\n' % flattened_js_url) + else: + output_file.write('<script>\n') + js = GenerateJS(load_sequence, minify=minify, report_sizes=report_sizes) + output_file.write(js) + output_file.write('</script>\n') + + for extra_script in extra_scripts: + extra_script.WriteToFile(output_file) + + if output_html_head_and_body: + output_file.write('</head>\n <body>\n </body>\n</html>\n') diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/generate_unittest.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/generate_unittest.py new file mode 100644 index 00000000000..1e83cb48c21 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/generate_unittest.py @@ -0,0 +1,89 @@ +# Copyright (c) 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import unittest + +from py_vulcanize import generate +from py_vulcanize import fake_fs +from py_vulcanize import project as project_module + + +class GenerateTests(unittest.TestCase): + + def setUp(self): + self.fs = fake_fs.FakeFS() + self.fs.AddFile( + '/x/foo/my_module.html', + ('<!DOCTYPE html>\n' + '<link rel="import" href="/foo/other_module.html">\n')) + self.fs.AddFile( + '/x/foo/other_module.html', + ('<!DOCTYPE html>\n' + '<script src="/foo/raw/raw_script.js"></script>\n' + '<script>\n' + ' \'use strict\';\n' + ' HelloWorld();\n' + '</script>\n')) + self.fs.AddFile('/x/foo/raw/raw_script.js', '\n/* raw script */\n') + self.fs.AddFile('/x/components/polymer/polymer.min.js', '\n') + + self.fs.AddFile('/x/foo/external_script.js', 'External()') + self.fs.AddFile('/x/foo/inline_and_external_module.html', + ('<!DOCTYPE html>\n' + '<script>Script1()</script>' + '<script src=/foo/external_script.js></script>' + '<script>Script2()</script>')) + + self.project = project_module.Project([os.path.normpath('/x')]) + + def testJSGeneration(self): + with self.fs: + load_sequence = self.project.CalcLoadSequenceForModuleNames( + [os.path.normpath('foo.my_module')]) + generate.GenerateJS(load_sequence) + + def testHTMLGeneration(self): + with self.fs: + load_sequence = self.project.CalcLoadSequenceForModuleNames( + [os.path.normpath('foo.my_module')]) + result = generate.GenerateStandaloneHTMLAsString(load_sequence) + self.assertIn('HelloWorld();', result) + + def testExtraScriptWithWriteContentsFunc(self): + with self.fs: + load_sequence = self.project.CalcLoadSequenceForModuleNames( + [os.path.normpath('foo.my_module')]) + + class ExtraScript(generate.ExtraScript): + + def WriteToFile(self, f): + f.write('<script>ExtraScript!</script>') + + result = generate.GenerateStandaloneHTMLAsString( + load_sequence, title='Title', extra_scripts=[ExtraScript()]) + self.assertIn('ExtraScript', result) + + def testScriptOrdering(self): + with self.fs: + load_sequence = self.project.CalcLoadSequenceForModuleNames( + [os.path.normpath('foo.inline_and_external_module')]) + result = generate.GenerateStandaloneHTMLAsString(load_sequence) + script1_pos = result.index('Script1()') + script2_pos = result.index('Script2()') + external_pos = result.index('External()') + self.assertTrue(script1_pos < external_pos < script2_pos) + + def testScriptOrderingWithIncludeTag(self): + with self.fs: + load_sequence = self.project.CalcLoadSequenceForModuleNames( + [os.path.normpath('foo.inline_and_external_module')]) + result = generate.GenerateJS(load_sequence, + use_include_tags_for_scripts = True, + dir_for_include_tag_root='/x/') + script1_pos = result.index('Script1()') + script2_pos = result.index('Script2()') + external_path = os.path.join('foo', 'external_script.js') + external_pos = result.index('<include src="{0}">'.format(external_path)) + self.assertTrue(script1_pos < external_pos < script2_pos) diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/html_generation_controller.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/html_generation_controller.py new file mode 100644 index 00000000000..c804fe8ca3e --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/html_generation_controller.py @@ -0,0 +1,28 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import re +from py_vulcanize import style_sheet + + +class HTMLGenerationController(object): + + def __init__(self): + self.current_module = None + + def GetHTMLForStylesheetHRef(self, href): # pylint: disable=unused-argument + return None + + def GetHTMLForInlineStylesheet(self, contents): + if self.current_module is None: + if re.search('url\(.+\)', contents): + raise Exception( + 'Default HTMLGenerationController cannot handle inline style urls') + return contents + + module_dirname = os.path.dirname(self.current_module.resource.absolute_path) + ss = style_sheet.ParsedStyleSheet( + self.current_module.loader, module_dirname, contents) + return ss.contents_with_inlined_images diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/html_module.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/html_module.py new file mode 100644 index 00000000000..5e1c7541c53 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/html_module.py @@ -0,0 +1,154 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import re + +from py_vulcanize import js_utils +from py_vulcanize import module +from py_vulcanize import parse_html_deps +from py_vulcanize import style_sheet + + +def IsHTMLResourceTheModuleGivenConflictingResourceNames( + js_resource, html_resource): # pylint: disable=unused-argument + return 'polymer-element' in html_resource.contents + + +class HTMLModule(module.Module): + + @property + def _module_dir_name(self): + return os.path.dirname(self.resource.absolute_path) + + def Parse(self, excluded_scripts): + try: + parser_results = parse_html_deps.HTMLModuleParser().Parse(self.contents) + except Exception as ex: + raise Exception('While parsing %s: %s' % (self.name, str(ex))) + + self.dependency_metadata = Parse(self.loader, + self.name, + self._module_dir_name, + self.IsThirdPartyComponent(), + parser_results, + excluded_scripts) + self._parser_results = parser_results + self.scripts = parser_results.scripts + + def Load(self, excluded_scripts): + super(HTMLModule, self).Load(excluded_scripts=excluded_scripts) + + reachable_names = set([m.name + for m in self.all_dependent_modules_recursive]) + if 'tr.exportTo' in self.contents: + if 'tracing.base.base' not in reachable_names: + raise Exception('%s: Does not have a dependency on base' % + os.path.relpath(self.resource.absolute_path)) + + for script in self.scripts: + if script.is_external: + if excluded_scripts and any(re.match(pattern, script.src) for + pattern in excluded_scripts): + continue + + resource = _HRefToResource(self.loader, self.name, self._module_dir_name, + script.src, + tag_for_err_msg='<script src="%s">' % script.src) + path = resource.unix_style_relative_path + raw_script = self.loader.LoadRawScript(path) + self.dependent_raw_scripts.append(raw_script) + script.loaded_raw_script = raw_script + + def GetTVCMDepsModuleType(self): + return 'py_vulcanize.HTML_MODULE_TYPE' + + def AppendHTMLContentsToFile(self, f, ctl, minify=False): + super(HTMLModule, self).AppendHTMLContentsToFile(f, ctl) + + ctl.current_module = self + try: + for piece in self._parser_results.YieldHTMLInPieces(ctl, minify=minify): + f.write(piece) + finally: + ctl.current_module = None + + def HRefToResource(self, href, tag_for_err_msg): + return _HRefToResource(self.loader, self.name, self._module_dir_name, + href, tag_for_err_msg) + + def AppendDirectlyDependentFilenamesTo( + self, dependent_filenames, include_raw_scripts=True): + super(HTMLModule, self).AppendDirectlyDependentFilenamesTo( + dependent_filenames, include_raw_scripts) + for contents in self._parser_results.inline_stylesheets: + module_dirname = os.path.dirname(self.resource.absolute_path) + ss = style_sheet.ParsedStyleSheet( + self.loader, module_dirname, contents) + ss.AppendDirectlyDependentFilenamesTo(dependent_filenames) + +def _HRefToResource( + loader, module_name, module_dir_name, href, tag_for_err_msg): + if href[0] == '/': + resource = loader.FindResourceGivenRelativePath( + os.path.normpath(href[1:])) + else: + abspath = os.path.normpath(os.path.join(module_dir_name, + os.path.normpath(href))) + resource = loader.FindResourceGivenAbsolutePath(abspath) + + if not resource: + raise module.DepsException( + 'In %s, the %s cannot be loaded because ' + 'it is not in the search path' % (module_name, tag_for_err_msg)) + try: + resource.contents + except: + raise module.DepsException('In %s, %s points at a nonexistent file ' % ( + module_name, tag_for_err_msg)) + return resource + + +def Parse(loader, module_name, module_dir_name, is_component, parser_results, + exclude_scripts=None): + res = module.ModuleDependencyMetadata() + if is_component: + return res + + # External script references. + for href in parser_results.scripts_external: + if exclude_scripts and any(re.match(pattern, href) for + pattern in exclude_scripts): + continue + + resource = _HRefToResource(loader, module_name, module_dir_name, + href, + tag_for_err_msg='<script src="%s">' % href) + res.dependent_raw_script_relative_paths.append( + resource.unix_style_relative_path) + + # External imports. Mostly the same as <script>, but we know its a module. + for href in parser_results.imports: + if exclude_scripts and any(re.match(pattern, href) for + pattern in exclude_scripts): + continue + + if not href.endswith('.html'): + raise Exception( + 'In %s, the <link rel="import" href="%s"> must point at a ' + 'file with an html suffix' % (module_name, href)) + + resource = _HRefToResource( + loader, module_name, module_dir_name, href, + tag_for_err_msg='<link rel="import" href="%s">' % href) + res.dependent_module_names.append(resource.name) + + # Style sheets. + for href in parser_results.stylesheets: + resource = _HRefToResource( + loader, module_name, module_dir_name, href, + tag_for_err_msg='<link rel="stylesheet" href="%s">' % href) + res.style_sheet_names.append(resource.name) + + return res diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/html_module_unittest.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/html_module_unittest.py new file mode 100644 index 00000000000..fb4af16c8d7 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/html_module_unittest.py @@ -0,0 +1,321 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import unittest +import StringIO + +from py_vulcanize import fake_fs +from py_vulcanize import generate +from py_vulcanize import html_generation_controller +from py_vulcanize import html_module +from py_vulcanize import parse_html_deps +from py_vulcanize import project as project_module +from py_vulcanize import resource +from py_vulcanize import resource_loader as resource_loader + + +class ResourceWithFakeContents(resource.Resource): + + def __init__(self, toplevel_dir, absolute_path, fake_contents): + """A resource with explicitly provided contents. + + If the resource does not exist, then pass fake_contents=None. This will + cause accessing the resource contents to raise an exception mimicking the + behavior of regular resources.""" + super(ResourceWithFakeContents, self).__init__(toplevel_dir, absolute_path) + self._fake_contents = fake_contents + + @property + def contents(self): + if self._fake_contents is None: + raise Exception('File not found') + return self._fake_contents + + +class FakeLoader(object): + + def __init__(self, source_paths, initial_filenames_and_contents=None): + self._source_paths = source_paths + self._file_contents = {} + if initial_filenames_and_contents: + for k, v in initial_filenames_and_contents.iteritems(): + self._file_contents[k] = v + + def FindResourceGivenAbsolutePath(self, absolute_path): + candidate_paths = [] + for source_path in self._source_paths: + if absolute_path.startswith(source_path): + candidate_paths.append(source_path) + if len(candidate_paths) == 0: + return None + + # Sort by length. Longest match wins. + candidate_paths.sort(lambda x, y: len(x) - len(y)) + longest_candidate = candidate_paths[-1] + + return ResourceWithFakeContents( + longest_candidate, absolute_path, + self._file_contents.get(absolute_path, None)) + + def FindResourceGivenRelativePath(self, relative_path): + absolute_path = None + for script_path in self._source_paths: + absolute_path = os.path.join(script_path, relative_path) + if absolute_path in self._file_contents: + return ResourceWithFakeContents(script_path, absolute_path, + self._file_contents[absolute_path]) + return None + + +class ParseTests(unittest.TestCase): + + def testValidExternalScriptReferenceToRawScript(self): + parse_results = parse_html_deps.HTMLModuleParserResults("""<!DOCTYPE html> + <script src="../foo.js"> + """) + + file_contents = {} + file_contents[os.path.normpath('/tmp/a/foo.js')] = """ +'i am just some raw script'; +""" + + metadata = html_module.Parse( + FakeLoader([os.path.normpath('/tmp')], file_contents), + 'a.b.start', + '/tmp/a/b/', + is_component=False, + parser_results=parse_results) + self.assertEquals([], metadata.dependent_module_names) + self.assertEquals( + ['a/foo.js'], metadata.dependent_raw_script_relative_paths) + + def testExternalScriptReferenceToModuleOutsideScriptPath(self): + parse_results = parse_html_deps.HTMLModuleParserResults("""<!DOCTYPE html> + <script src="/foo.js"> + """) + + file_contents = {} + file_contents[os.path.normpath('/foo.js')] = '' + + def DoIt(): + html_module.Parse(FakeLoader([os.path.normpath('/tmp')], file_contents), + 'a.b.start', + '/tmp/a/b/', + is_component=False, + parser_results=parse_results) + self.assertRaises(Exception, DoIt) + + def testExternalScriptReferenceToFileThatDoesntExist(self): + parse_results = parse_html_deps.HTMLModuleParserResults("""<!DOCTYPE html> + <script src="/foo.js"> + """) + + file_contents = {} + + def DoIt(): + html_module.Parse(FakeLoader([os.path.normpath('/tmp')], file_contents), + 'a.b.start', + '/tmp/a/b/', + is_component=False, + parser_results=parse_results) + self.assertRaises(Exception, DoIt) + + def testValidImportOfModule(self): + parse_results = parse_html_deps.HTMLModuleParserResults("""<!DOCTYPE html> + <link rel="import" href="../foo.html"> + """) + + file_contents = {} + file_contents[os.path.normpath('/tmp/a/foo.html')] = """ +""" + + metadata = html_module.Parse( + FakeLoader([os.path.normpath('/tmp')], file_contents), + 'a.b.start', + '/tmp/a/b/', + is_component=False, + parser_results=parse_results) + self.assertEquals(['a.foo'], metadata.dependent_module_names) + + def testStyleSheetImport(self): + parse_results = parse_html_deps.HTMLModuleParserResults("""<!DOCTYPE html> + <link rel="stylesheet" href="../foo.css"> + """) + + file_contents = {} + file_contents[os.path.normpath('/tmp/a/foo.css')] = """ +""" + metadata = html_module.Parse( + FakeLoader([os.path.normpath('/tmp')], file_contents), + 'a.b.start', + '/tmp/a/b/', + is_component=False, + parser_results=parse_results) + self.assertEquals([], metadata.dependent_module_names) + self.assertEquals(['a.foo'], metadata.style_sheet_names) + + def testUsingAbsoluteHref(self): + parse_results = parse_html_deps.HTMLModuleParserResults("""<!DOCTYPE html> + <script src="/foo.js"> + """) + + file_contents = {} + file_contents[os.path.normpath('/src/foo.js')] = '' + + metadata = html_module.Parse( + FakeLoader([os.path.normpath("/tmp"), os.path.normpath("/src")], + file_contents), + "a.b.start", + "/tmp/a/b/", + is_component=False, + parser_results=parse_results) + self.assertEquals(['foo.js'], metadata.dependent_raw_script_relative_paths) + + +class HTMLModuleTests(unittest.TestCase): + + def testBasicModuleGeneration(self): + file_contents = {} + file_contents[os.path.normpath('/tmp/a/b/start.html')] = """ +<!DOCTYPE html> +<link rel="import" href="/widget.html"> +<link rel="stylesheet" href="../common.css"> +<script src="/raw_script.js"></script> +<script src="/excluded_script.js"></script> +<dom-module id="start"> + <template> + </template> + <script> + 'use strict'; + console.log('inline script for start.html got written'); + </script> +</dom-module> +""" + file_contents[os.path.normpath('/py_vulcanize/py_vulcanize.html')] = """<!DOCTYPE html> +""" + file_contents[os.path.normpath('/components/widget.html')] = """ +<!DOCTYPE html> +<link rel="import" href="/py_vulcanize.html"> +<widget name="widget.html"></widget> +<script> +'use strict'; +console.log('inline script for widget.html'); +</script> +""" + file_contents[os.path.normpath('/tmp/a/common.css')] = """ +/* /tmp/a/common.css was written */ +""" + file_contents[os.path.normpath('/raw/raw_script.js')] = """ +console.log('/raw/raw_script.js was written'); +""" + file_contents[os.path.normpath( + '/raw/components/polymer/polymer.min.js')] = """ +""" + + with fake_fs.FakeFS(file_contents): + project = project_module.Project( + [os.path.normpath('/py_vulcanize/'), + os.path.normpath('/tmp/'), + os.path.normpath('/components/'), + os.path.normpath('/raw/')]) + loader = resource_loader.ResourceLoader(project) + a_b_start_module = loader.LoadModule( + module_name='a.b.start', excluded_scripts=['\/excluded_script.js']) + load_sequence = project.CalcLoadSequenceForModules([a_b_start_module]) + + # Check load sequence names. + load_sequence_names = [x.name for x in load_sequence] + self.assertEquals(['py_vulcanize', + 'widget', + 'a.b.start'], load_sequence_names) + + # Check module_deps on a_b_start_module + def HasDependentModule(module, name): + return [x for x in module.dependent_modules + if x.name == name] + assert HasDependentModule(a_b_start_module, 'widget') + + # Check JS generation. + js = generate.GenerateJS(load_sequence) + assert 'inline script for start.html' in js + assert 'inline script for widget.html' in js + assert '/raw/raw_script.js' in js + assert 'excluded_script.js' not in js + + # Check HTML generation. + html = generate.GenerateStandaloneHTMLAsString( + load_sequence, title='', flattened_js_url='/blah.js') + assert '<dom-module id="start">' in html + assert 'inline script for widget.html' not in html + assert 'common.css' in html + + def testPolymerConversion(self): + file_contents = {} + file_contents[os.path.normpath('/tmp/a/b/my_component.html')] = """ +<!DOCTYPE html> +<dom-module id="my-component"> + <template> + </template> + <script> + 'use strict'; + Polymer ( { + is: "my-component" + }); + </script> +</dom-module> +""" + with fake_fs.FakeFS(file_contents): + project = project_module.Project([ + os.path.normpath('/py_vulcanize/'), os.path.normpath('/tmp/')]) + loader = resource_loader.ResourceLoader(project) + my_component = loader.LoadModule(module_name='a.b.my_component') + + f = StringIO.StringIO() + my_component.AppendJSContentsToFile( + f, + use_include_tags_for_scripts=False, + dir_for_include_tag_root=None) + js = f.getvalue().rstrip() + expected_js = """ + 'use strict'; + Polymer ( { + is: "my-component" + }); +""".rstrip() + self.assertEquals(expected_js, js) + + def testInlineStylesheetURLs(self): + file_contents = {} + file_contents[os.path.normpath('/tmp/a/b/my_component.html')] = """ +<!DOCTYPE html> +<style> +.some-rule { + background-image: url('../something.jpg'); +} +</style> +""" + file_contents[os.path.normpath('/tmp/a/something.jpg')] = 'jpgdata' + with fake_fs.FakeFS(file_contents): + project = project_module.Project([ + os.path.normpath('/py_vulcanize/'), os.path.normpath('/tmp/')]) + loader = resource_loader.ResourceLoader(project) + my_component = loader.LoadModule(module_name='a.b.my_component') + + computed_deps = [] + my_component.AppendDirectlyDependentFilenamesTo(computed_deps) + self.assertEquals(set(computed_deps), + set([os.path.normpath('/tmp/a/b/my_component.html'), + os.path.normpath('/tmp/a/something.jpg')])) + + f = StringIO.StringIO() + ctl = html_generation_controller.HTMLGenerationController() + my_component.AppendHTMLContentsToFile(f, ctl) + html = f.getvalue().rstrip() + # FIXME: This is apparently not used. + expected_html = """ +.some-rule { + background-image: url(data:image/jpg;base64,anBnZGF0YQ==); +} +""".rstrip() diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/js_utils.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/js_utils.py new file mode 100644 index 00000000000..6e6ca9db674 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/js_utils.py @@ -0,0 +1,7 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +def EscapeJSIfNeeded(js): + return js.replace('</script>', '<\/script>') diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/js_utils_unittest.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/js_utils_unittest.py new file mode 100644 index 00000000000..cb8025c97db --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/js_utils_unittest.py @@ -0,0 +1,18 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import unittest + +from py_vulcanize import js_utils + + +class ValidateStrictModeTests(unittest.TestCase): + + def testEscapeJSIfNeeded(self): + self.assertEqual( + '<script>var foo;<\/script>', + js_utils.EscapeJSIfNeeded('<script>var foo;</script>')) + self.assertEqual( + '<script>var foo;<\/script>', + js_utils.EscapeJSIfNeeded('<script>var foo;<\/script>')) diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/module.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/module.py new file mode 100644 index 00000000000..bd6a68fa442 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/module.py @@ -0,0 +1,262 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""This module contains the Module class and other classes for resources. + +The Module class represents a module in the trace viewer system. A module has +a name, and may require a variety of other resources, such as stylesheets, +template objects, raw JavaScript, or other modules. + +Other resources include HTML templates, raw JavaScript files, and stylesheets. +""" + +import os +import inspect +import codecs + +from py_vulcanize import js_utils + + +class DepsException(Exception): + """Exceptions related to module dependency resolution.""" + + def __init__(self, fmt, *args): + from py_vulcanize import style_sheet as style_sheet_module + context = [] + frame = inspect.currentframe() + while frame: + frame_locals = frame.f_locals + + module_name = None + if 'self' in frame_locals: + s = frame_locals['self'] + if isinstance(s, Module): + module_name = s.name + if isinstance(s, style_sheet_module.StyleSheet): + module_name = s.name + '.css' + if not module_name: + if 'module' in frame_locals: + module = frame_locals['module'] + if isinstance(s, Module): + module_name = module.name + elif 'm' in frame_locals: + module = frame_locals['m'] + if isinstance(s, Module): + module_name = module.name + + if module_name: + if len(context): + if context[-1] != module_name: + context.append(module_name) + else: + context.append(module_name) + + frame = frame.f_back + + context.reverse() + self.context = context + context_str = '\n'.join(' %s' % x for x in context) + Exception.__init__( + self, 'While loading:\n%s\nGot: %s' % (context_str, (fmt % args))) + + +class ModuleDependencyMetadata(object): + + def __init__(self): + self.dependent_module_names = [] + self.dependent_raw_script_relative_paths = [] + self.style_sheet_names = [] + + def AppendMetdata(self, other): + self.dependent_module_names += other.dependent_module_names + self.dependent_raw_script_relative_paths += \ + other.dependent_raw_script_relative_paths + self.style_sheet_names += other.style_sheet_names + + +_next_module_id = 1 + + +class Module(object): + """Represents a JavaScript module. + + Interesting properties include: + name: Module name, may include a namespace, e.g. 'py_vulcanize.foo'. + filename: The filename of the actual module. + contents: The text contents of the module. + dependent_modules: Other modules that this module depends on. + + In addition to these properties, a Module also contains lists of other + resources that it depends on. + """ + + def __init__(self, loader, name, resource, load_resource=True): + assert isinstance(name, basestring), 'Got %s instead' % repr(name) + + global _next_module_id + self._id = _next_module_id + _next_module_id += 1 + + self.loader = loader + self.name = name + self.resource = resource + + if load_resource: + f = codecs.open(self.filename, mode='r', encoding='utf-8') + self.contents = f.read() + f.close() + else: + self.contents = None + + # Dependency metadata, set up during Parse(). + self.dependency_metadata = None + + # Actual dependencies, set up during load(). + self.dependent_modules = [] + self.dependent_raw_scripts = [] + self.scripts = [] + self.style_sheets = [] + + # Caches. + self._all_dependent_modules_recursive = None + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, self.name) + + @property + def id(self): + return self._id + + @property + def filename(self): + return self.resource.absolute_path + + def IsThirdPartyComponent(self): + """Checks whether this module is a third-party Polymer component.""" + if os.path.join('third_party', 'components') in self.filename: + return True + if os.path.join('third_party', 'polymer', 'components') in self.filename: + return True + return False + + def Parse(self, excluded_scripts): + """Parses self.contents and fills in the module's dependency metadata.""" + raise NotImplementedError() + + def GetTVCMDepsModuleType(self): + """Returns the py_vulcanize.setModuleInfo type for this module""" + raise NotImplementedError() + + def AppendJSContentsToFile(self, + f, + use_include_tags_for_scripts, + dir_for_include_tag_root): + """Appends the js for this module to the provided file.""" + for script in self.scripts: + script.AppendJSContentsToFile(f, use_include_tags_for_scripts, + dir_for_include_tag_root) + + def AppendHTMLContentsToFile(self, f, ctl, minify=False): + """Appends the HTML for this module [without links] to the provided file.""" + pass + + def Load(self, excluded_scripts=None): + """Loads the sub-resources that this module depends on from its dependency + metadata. + + Raises: + DepsException: There was a problem finding one of the dependencies. + Exception: There was a problem parsing a module that this one depends on. + """ + assert self.name, 'Module name must be set before dep resolution.' + assert self.filename, 'Module filename must be set before dep resolution.' + assert self.name in self.loader.loaded_modules, ( + 'Module must be registered in resource loader before loading.') + + metadata = self.dependency_metadata + for name in metadata.dependent_module_names: + module = self.loader.LoadModule(module_name=name, + excluded_scripts=excluded_scripts) + self.dependent_modules.append(module) + + for name in metadata.style_sheet_names: + style_sheet = self.loader.LoadStyleSheet(name) + self.style_sheets.append(style_sheet) + + @property + def all_dependent_modules_recursive(self): + if self._all_dependent_modules_recursive: + return self._all_dependent_modules_recursive + + self._all_dependent_modules_recursive = set(self.dependent_modules) + for dependent_module in self.dependent_modules: + self._all_dependent_modules_recursive.update( + dependent_module.all_dependent_modules_recursive) + return self._all_dependent_modules_recursive + + def ComputeLoadSequenceRecursive(self, load_sequence, already_loaded_set, + depth=0): + """Recursively builds up a load sequence list. + + Args: + load_sequence: A list which will be incrementally built up. + already_loaded_set: A set of modules that has already been added to the + load sequence list. + depth: The depth of recursion. If it too deep, that indicates a loop. + """ + if depth > 32: + raise Exception('Include loop detected on %s', self.name) + for dependent_module in self.dependent_modules: + if dependent_module.name in already_loaded_set: + continue + dependent_module.ComputeLoadSequenceRecursive( + load_sequence, already_loaded_set, depth + 1) + if self.name not in already_loaded_set: + already_loaded_set.add(self.name) + load_sequence.append(self) + + def GetAllDependentFilenamesRecursive(self, include_raw_scripts=True): + dependent_filenames = [] + + visited_modules = set() + + def Get(module): + module.AppendDirectlyDependentFilenamesTo( + dependent_filenames, include_raw_scripts) + visited_modules.add(module) + for m in module.dependent_modules: + if m in visited_modules: + continue + Get(m) + + Get(self) + return dependent_filenames + + def AppendDirectlyDependentFilenamesTo( + self, dependent_filenames, include_raw_scripts=True): + dependent_filenames.append(self.resource.absolute_path) + if include_raw_scripts: + for raw_script in self.dependent_raw_scripts: + dependent_filenames.append(raw_script.resource.absolute_path) + for style_sheet in self.style_sheets: + style_sheet.AppendDirectlyDependentFilenamesTo(dependent_filenames) + + +class RawScript(object): + """Represents a raw script resource referenced by a module via the + py_vulcanize.requireRawScript(xxx) directive.""" + + def __init__(self, resource): + self.resource = resource + + @property + def filename(self): + return self.resource.absolute_path + + @property + def contents(self): + return self.resource.contents + + def __repr__(self): + return 'RawScript(%s)' % self.filename diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/module_unittest.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/module_unittest.py new file mode 100644 index 00000000000..ed92db35362 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/module_unittest.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for the module module, which contains Module and related classes.""" + +import os +import unittest + +from py_vulcanize import fake_fs +from py_vulcanize import module +from py_vulcanize import resource_loader +from py_vulcanize import project as project_module + + +class ModuleIntegrationTests(unittest.TestCase): + + def test_module(self): + fs = fake_fs.FakeFS() + fs.AddFile('/src/x.html', """ +<!DOCTYPE html> +<link rel="import" href="/y.html"> +<link rel="import" href="/z.html"> +<script> +'use strict'; +</script> +""") + fs.AddFile('/src/y.html', """ +<!DOCTYPE html> +<link rel="import" href="/z.html"> +""") + fs.AddFile('/src/z.html', """ +<!DOCTYPE html> +""") + fs.AddFile('/src/py_vulcanize.html', '<!DOCTYPE html>') + with fs: + project = project_module.Project([os.path.normpath('/src/')]) + loader = resource_loader.ResourceLoader(project) + x_module = loader.LoadModule('x') + + self.assertEquals([loader.loaded_modules['y'], + loader.loaded_modules['z']], + x_module.dependent_modules) + + already_loaded_set = set() + load_sequence = [] + x_module.ComputeLoadSequenceRecursive(load_sequence, already_loaded_set) + + self.assertEquals([loader.loaded_modules['z'], + loader.loaded_modules['y'], + x_module], + load_sequence) + + def testBasic(self): + fs = fake_fs.FakeFS() + fs.AddFile('/x/src/my_module.html', """ +<!DOCTYPE html> +<link rel="import" href="/py_vulcanize/foo.html"> +}); +""") + fs.AddFile('/x/py_vulcanize/foo.html', """ +<!DOCTYPE html> +}); +""") + project = project_module.Project([os.path.normpath('/x')]) + loader = resource_loader.ResourceLoader(project) + with fs: + my_module = loader.LoadModule(module_name='src.my_module') + dep_names = [x.name for x in my_module.dependent_modules] + self.assertEquals(['py_vulcanize.foo'], dep_names) + + def testDepsExceptionContext(self): + fs = fake_fs.FakeFS() + fs.AddFile('/x/src/my_module.html', """ +<!DOCTYPE html> +<link rel="import" href="/py_vulcanize/foo.html"> +""") + fs.AddFile('/x/py_vulcanize/foo.html', """ +<!DOCTYPE html> +<link rel="import" href="missing.html"> +""") + project = project_module.Project([os.path.normpath('/x')]) + loader = resource_loader.ResourceLoader(project) + with fs: + exc = None + try: + loader.LoadModule(module_name='src.my_module') + assert False, 'Expected an exception' + except module.DepsException as e: + exc = e + self.assertEquals( + ['src.my_module', 'py_vulcanize.foo'], + exc.context) + + def testGetAllDependentFilenamesRecursive(self): + fs = fake_fs.FakeFS() + fs.AddFile('/x/y/z/foo.html', """ +<!DOCTYPE html> +<link rel="import" href="/z/foo2.html"> +<link rel="stylesheet" href="/z/foo.css"> +<script src="/bar.js"></script> +""") + fs.AddFile('/x/y/z/foo.css', """ +.x .y { + background-image: url(foo.jpeg); +} +""") + fs.AddFile('/x/y/z/foo.jpeg', '') + fs.AddFile('/x/y/z/foo2.html', """ +<!DOCTYPE html> +""") + fs.AddFile('/x/raw/bar.js', 'hello') + project = project_module.Project([ + os.path.normpath('/x/y'), os.path.normpath('/x/raw/')]) + loader = resource_loader.ResourceLoader(project) + with fs: + my_module = loader.LoadModule(module_name='z.foo') + self.assertEquals(1, len(my_module.dependent_raw_scripts)) + + dependent_filenames = my_module.GetAllDependentFilenamesRecursive() + self.assertEquals( + [ + os.path.normpath('/x/y/z/foo.html'), + os.path.normpath('/x/raw/bar.js'), + os.path.normpath('/x/y/z/foo.css'), + os.path.normpath('/x/y/z/foo.jpeg'), + os.path.normpath('/x/y/z/foo2.html'), + ], + dependent_filenames) diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/parse_html_deps.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/parse_html_deps.py new file mode 100644 index 00000000000..4a0888ca1dc --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/parse_html_deps.py @@ -0,0 +1,283 @@ +# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import sys + +from py_vulcanize import js_utils +from py_vulcanize import module +from py_vulcanize import strip_js_comments +from py_vulcanize import html_generation_controller + + +def _AddToPathIfNeeded(path): + if path not in sys.path: + sys.path.insert(0, path) + + +def _InitBeautifulSoup(): + catapult_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), + os.path.pardir, os.path.pardir, os.path.pardir)) + bs_path = os.path.join(catapult_path, 'third_party', 'beautifulsoup4') + _AddToPathIfNeeded(bs_path) + + html5lib_path = os.path.join(catapult_path, 'third_party', 'html5lib-python') + _AddToPathIfNeeded(html5lib_path) + + six_path = os.path.join(catapult_path, 'third_party', 'six') + _AddToPathIfNeeded(six_path) + + +_InitBeautifulSoup() +import bs4 + +class Script(object): + + def __init__(self, soup): + if not soup: + raise module.DepsException('Script object created without soup') + self._soup = soup + + def AppendJSContentsToFile(self, f, *args, **kwargs): + raise NotImplementedError() + +class InlineScript(Script): + + def __init__(self, soup): + super(InlineScript, self).__init__(soup) + self._stripped_contents = None + self._open_tags = None + self.is_external = False + + @property + def contents(self): + return unicode(self._soup.string) + + @property + def stripped_contents(self): + if not self._stripped_contents: + self._stripped_contents = strip_js_comments.StripJSComments( + self.contents) + return self._stripped_contents + + @property + def open_tags(self): + if self._open_tags: + return self._open_tags + open_tags = [] + cur = self._soup.parent + while cur: + if isinstance(cur, bs4.BeautifulSoup): + break + + open_tags.append(_Tag(cur.name, cur.attrs)) + cur = cur.parent + + open_tags.reverse() + assert open_tags[-1].tag == 'script' + del open_tags[-1] + + self._open_tags = open_tags + return self._open_tags + + def AppendJSContentsToFile(self, f, *args, **kwargs): + js = self.contents + escaped_js = js_utils.EscapeJSIfNeeded(js) + f.write(escaped_js) + f.write('\n') + +class ExternalScript(Script): + + def __init__(self, soup): + super(ExternalScript, self).__init__(soup) + if 'src' not in soup.attrs: + raise Exception("{0} is not an external script.".format(soup)) + self.is_external = True + self._loaded_raw_script = None + + @property + def loaded_raw_script(self): + if self._loaded_raw_script: + return self._loaded_raw_script + + return None + + @loaded_raw_script.setter + def loaded_raw_script(self, value): + self._loaded_raw_script = value + + @property + def src(self): + return self._soup.attrs['src'] + + def AppendJSContentsToFile(self, + f, + use_include_tags_for_scripts, + dir_for_include_tag_root): + raw_script = self.loaded_raw_script + if not raw_script: + return + + if use_include_tags_for_scripts: + rel_filename = os.path.relpath(raw_script.filename, + dir_for_include_tag_root) + f.write("""<include src="%s">\n""" % rel_filename) + else: + f.write(js_utils.EscapeJSIfNeeded(raw_script.contents)) + f.write('\n') + +def _CreateSoupWithoutHeadOrBody(html): + soupCopy = bs4.BeautifulSoup(html, 'html5lib') + soup = bs4.BeautifulSoup() + soup.reset() + if soupCopy.head: + for n in soupCopy.head.contents: + n.extract() + soup.append(n) + if soupCopy.body: + for n in soupCopy.body.contents: + n.extract() + soup.append(n) + return soup + + +class HTMLModuleParserResults(object): + + def __init__(self, html): + self._soup = bs4.BeautifulSoup(html, 'html5lib') + self._inline_scripts = None + self._scripts = None + + @property + def scripts_external(self): + tags = self._soup.findAll('script', src=True) + return [t['src'] for t in tags] + + @property + def inline_scripts(self): + if not self._inline_scripts: + tags = self._soup.findAll('script', src=None) + self._inline_scripts = [InlineScript(t.string) for t in tags] + return self._inline_scripts + + @property + def scripts(self): + if not self._scripts: + self._scripts = [] + script_elements = self._soup.findAll('script') + for element in script_elements: + if 'src' in element.attrs: + self._scripts.append(ExternalScript(element)) + else: + self._scripts.append(InlineScript(element)) + return self._scripts + + @property + def imports(self): + tags = self._soup.findAll('link', rel='import') + return [t['href'] for t in tags] + + @property + def stylesheets(self): + tags = self._soup.findAll('link', rel='stylesheet') + return [t['href'] for t in tags] + + @property + def inline_stylesheets(self): + tags = self._soup.findAll('style') + return [unicode(t.string) for t in tags] + + def YieldHTMLInPieces(self, controller, minify=False): + yield self.GenerateHTML(controller, minify) + + def GenerateHTML(self, controller, minify=False, prettify=False): + soup = _CreateSoupWithoutHeadOrBody(unicode(self._soup)) + + # Remove declaration. + for x in soup.contents: + if isinstance(x, bs4.Doctype): + x.extract() + + # Remove declaration. + for x in soup.contents: + if isinstance(x, bs4.Declaration): + x.extract() + + # Remove all imports. + imports = soup.findAll('link', rel='import') + for imp in imports: + imp.extract() + + # Remove all script links. + scripts_external = soup.findAll('script', src=True) + for script in scripts_external: + script.extract() + + # Remove all in-line scripts. + scripts_external = soup.findAll('script', src=None) + for script in scripts_external: + script.extract() + + # Process all in-line styles. + inline_styles = soup.findAll('style') + for style in inline_styles: + html = controller.GetHTMLForInlineStylesheet(unicode(style.string)) + if html: + ns = soup.new_tag('style') + ns.append(bs4.NavigableString(html)) + style.replaceWith(ns) + else: + style.extract() + + # Rewrite all external stylesheet hrefs or remove, as needed. + stylesheet_links = soup.findAll('link', rel='stylesheet') + for stylesheet_link in stylesheet_links: + html = controller.GetHTMLForStylesheetHRef(stylesheet_link['href']) + if html: + tmp = bs4.BeautifulSoup(html, 'html5lib').findAll('style') + assert len(tmp) == 1 + stylesheet_link.replaceWith(tmp[0]) + else: + stylesheet_link.extract() + + # Remove comments if minifying. + if minify: + comments = soup.findAll( + text=lambda text: isinstance(text, bs4.Comment)) + for comment in comments: + comment.extract() + if prettify: + return soup.prettify('utf-8').strip() + + # We are done. + return unicode(soup).strip() + + @property + def html_contents_without_links_and_script(self): + return self.GenerateHTML( + html_generation_controller.HTMLGenerationController()) + + +class _Tag(object): + + def __init__(self, tag, attrs): + self.tag = tag + self.attrs = attrs + + def __repr__(self): + attr_string = ' '.join('%s="%s"' % (x[0], x[1]) for x in self.attrs) + return '<%s %s>' % (self.tag, attr_string) + + +class HTMLModuleParser(): + + def Parse(self, html): + if html is None: + html = '' + else: + if html.find('< /script>') != -1: + raise Exception('Escape script tags with <\/script>') + + return HTMLModuleParserResults(html) diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/parse_html_deps_unittest.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/parse_html_deps_unittest.py new file mode 100755 index 00000000000..2a30a29b059 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/parse_html_deps_unittest.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python +# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import re +import unittest + +from py_vulcanize import parse_html_deps +from py_vulcanize import html_generation_controller + + +class ParseTests(unittest.TestCase): + + def test_parse_empty(self): + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse('') + self.assertEquals([], module.scripts_external) + self.assertEquals([], module.inline_scripts) + self.assertEquals([], module.stylesheets) + self.assertEquals([], module.imports) + + def test_parse_none(self): + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(None) + self.assertEquals([], module.scripts_external) + self.assertEquals([], module.inline_scripts) + self.assertEquals([], module.stylesheets) + self.assertEquals([], module.imports) + + def test_parse_script_src_basic(self): + html = """<!DOCTYPE html> + <html> + <head> + <script src="polymer.min.js"></script> + <script src="foo.js"></script> + </head> + <body> + </body> + </html>""" + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals(['polymer.min.js', 'foo.js'], module.scripts_external) + self.assertEquals([], module.inline_scripts) + self.assertEquals([], module.stylesheets) + self.assertEquals([], module.imports) + self.assertNotIn( + 'DOCTYPE html', + module.html_contents_without_links_and_script) + + def test_parse_link_rel_import(self): + html = """<!DOCTYPE html> + <html> + <head> + <link rel="import" href="x-foo.html"> + </head> + <body> + </body> + </html>""" + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals([], module.scripts_external) + self.assertEquals([], module.inline_scripts) + self.assertEquals([], module.stylesheets) + self.assertEquals(['x-foo.html'], module.imports) + + def test_parse_script_inline(self): + html = """<polymer-element name="tk-element-proto"> + <template> + </template> + <script> + py_vulcanize.require("foo"); + py_vulcanize.require('bar'); + </script> + </polymer-element>""" + + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals([], module.scripts_external) + self.assertEquals(1, len(module.inline_scripts)) + self.assertEquals([], module.stylesheets) + self.assertEquals([], module.imports) + + script0 = module.inline_scripts[0] + val = re.sub(r'\s+', '', script0.contents) + inner_script = """py_vulcanize.require("foo");py_vulcanize.require('bar');""" + self.assertEquals(inner_script, val) + + self.assertEquals(3, len(script0.open_tags)) + self.assertEquals('polymer-element', script0.open_tags[2].tag) + + self.assertNotIn( + 'py_vulcanize.require("foo");', + module.html_contents_without_links_and_script) + + def test_parse_script_inline_and_external(self): + html = """<polymer-element name="tk-element-proto"> + <template> + </template> + <script>window = {}</script> + <script src="foo.js"></script> + <script>window = undefined</script> + </polymer-element>""" + + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals(3, len(module.scripts)) + self.assertEquals('window = {}', module.scripts[0].contents) + self.assertEquals("foo.js",module.scripts[1].src) + self.assertTrue(module.scripts[1].is_external) + self.assertEquals('window = undefined', module.scripts[2].contents) + self.assertEquals([], module.imports) + + def test_parse_script_src_sripping(self): + html = """ +<script src="blah.js"></script> +""" + module = parse_html_deps.HTMLModuleParser().Parse(html) + self.assertEquals('', + module.html_contents_without_links_and_script) + + def test_parse_link_rel_stylesheet(self): + html = """<polymer-element name="hi"> + <template> + <link rel="stylesheet" href="frameworkstyles.css"> + </template> + </polymer-element>""" + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals([], module.scripts_external) + self.assertEquals([], module.inline_scripts) + self.assertEquals(['frameworkstyles.css'], module.stylesheets) + self.assertEquals([], module.imports) + + class Ctl(html_generation_controller.HTMLGenerationController): + + def GetHTMLForStylesheetHRef(self, href): + if href == 'frameworkstyles.css': + return '<style>FRAMEWORK</style>' + return None + + gen_html = module.GenerateHTML(Ctl()) + ghtm = """<polymer-element name="hi"> + <template> + <style>FRAMEWORK</style> + </template> + </polymer-element>""" + self.assertEquals(ghtm, gen_html) + + def test_parse_inline_style(self): + html = """<style> + hello +</style>""" + module = parse_html_deps.HTMLModuleParser().Parse(html) + self.assertEquals(html, module.html_contents_without_links_and_script) + + class Ctl(html_generation_controller.HTMLGenerationController): + + def GetHTMLForInlineStylesheet(self, contents): + if contents == '\n hello\n': + return '\n HELLO\n' + return None + + gen_html = module.GenerateHTML(Ctl()) + ghtm = """<style> + HELLO +</style>""" + self.assertEquals(ghtm, gen_html) + + def test_parse_style_import(self): + html = """<polymer-element name="x-blink"> + <template> + <style> + @import url(awesome.css); + </style> + </template> + </polymer-element>""" + parser = parse_html_deps.HTMLModuleParser() + self.assertRaises(lambda: parser.Parse(html)) + + def test_nested_templates(self): + orig_html = """<template> + <template> + <div id="foo"></div> + </template> + </template>""" + parser = parse_html_deps.HTMLModuleParser() + res = parser.Parse(orig_html) + html = res.html_contents_without_links_and_script + self.assertEquals(html, orig_html) + + def test_html_contents_basic(self): + html = """<a b="c">d</a>""" + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals(html, module.html_contents_without_links_and_script) + + def test_html_contents_with_entity(self): + html = """<a>→</a>""" + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals(u'<a>\u2192</a>', + module.html_contents_without_links_and_script) + + def test_html_content_with_charref(self): + html = """<a>></a>""" + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals('<a>></a>', + module.html_contents_without_links_and_script) + + def test_html_content_start_end_br(self): + html = """<a><br /></a>""" + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals('<a><br/></a>', + module.html_contents_without_links_and_script) + + def test_html_content_start_end_img(self): + html = """<a><img src="foo.png" id="bar" /></a>""" + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals('<a><img id="bar" src="foo.png"/></a>', + module.html_contents_without_links_and_script) + + def test_html_contents_with_link_stripping(self): + html = """<a b="c">d</a> + <link rel="import" href="x-foo.html">""" + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals("""<a b="c">d</a>""", + module.html_contents_without_links_and_script.strip()) + + def test_html_contents_with_style_link_stripping(self): + html = """<a b="c">d</a> + <link rel="stylesheet" href="frameworkstyles.css">""" + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals("""<a b="c">d</a>""", + module.html_contents_without_links_and_script.strip()) + + def test_br_does_not_raise(self): + html = """<div><br/></div>""" + parser = parse_html_deps.HTMLModuleParser() + parser.Parse(html) + + def test_p_does_not_raises(self): + html = """<div></p></div>""" + parser = parse_html_deps.HTMLModuleParser() + parser.Parse(html) + + def test_link_endlink_does_not_raise(self): + html = """<link rel="stylesheet" href="foo.css"></link>""" + parser = parse_html_deps.HTMLModuleParser() + parser.Parse(html) + + def test_link_script_does_not_raise(self): + html = """<link rel="stylesheet" href="foo.css"> + <script> + </script>""" + parser = parse_html_deps.HTMLModuleParser() + parser.Parse(html) + + def test_script_with_script_inside_as_js(self): + html = """<script> + var html_lines = [ + '<script>', + '<\/script>', + ]; + </script>""" + parser = parse_html_deps.HTMLModuleParser() + parser.Parse(html) + + def test_invalid_script_escaping_raises(self): + html = """<script> + var html_lines = [ + '<script>', + '< /script>', + ]; + </script>""" + parser = parse_html_deps.HTMLModuleParser() + + def DoIt(): + parser.Parse(html) + self.assertRaises(Exception, DoIt) + + def test_script_with_cdata(self): + html = """<script></h2></script>""" + parser = parse_html_deps.HTMLModuleParser() + module = parser.Parse(html) + self.assertEquals(1, len(module.inline_scripts)) + self.assertEquals('</h2>', module.inline_scripts[0].contents) diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/project.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/project.py new file mode 100644 index 00000000000..6b65784eb0e --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/project.py @@ -0,0 +1,235 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import collections +import os + +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + +from py_vulcanize import resource_loader + + +def _FindAllFilesRecursive(source_paths): + all_filenames = set() + for source_path in source_paths: + for dirpath, _, filenames in os.walk(source_path): + for f in filenames: + if f.startswith('.'): + continue + x = os.path.abspath(os.path.join(dirpath, f)) + all_filenames.add(x) + return all_filenames + + +class AbsFilenameList(object): + + def __init__(self, willDirtyCallback): + self._willDirtyCallback = willDirtyCallback + self._filenames = [] + self._filenames_set = set() + + def _WillBecomeDirty(self): + if self._willDirtyCallback: + self._willDirtyCallback() + + def append(self, filename): + assert os.path.isabs(filename) + self._WillBecomeDirty() + self._filenames.append(filename) + self._filenames_set.add(filename) + + def extend(self, iterable): + self._WillBecomeDirty() + for filename in iterable: + assert os.path.isabs(filename) + self._filenames.append(filename) + self._filenames_set.add(filename) + + def appendRel(self, basedir, filename): + assert os.path.isabs(basedir) + self._WillBecomeDirty() + n = os.path.abspath(os.path.join(basedir, filename)) + self._filenames.append(n) + self._filenames_set.add(n) + + def extendRel(self, basedir, iterable): + self._WillBecomeDirty() + assert os.path.isabs(basedir) + for filename in iterable: + n = os.path.abspath(os.path.join(basedir, filename)) + self._filenames.append(n) + self._filenames_set.add(n) + + def __contains__(self, x): + return x in self._filenames_set + + def __len__(self): + return self._filenames.__len__() + + def __iter__(self): + return iter(self._filenames) + + def __repr__(self): + return repr(self._filenames) + + def __str__(self): + return str(self._filenames) + + +class Project(object): + + py_vulcanize_path = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..')) + + def __init__(self, source_paths=None): + """ + source_paths: A list of top-level directories in which modules and raw + scripts can be found. Module paths are relative to these directories. + """ + self._loader = None + self._frozen = False + self.source_paths = AbsFilenameList(self._WillPartOfPathChange) + + if source_paths is not None: + self.source_paths.extend(source_paths) + + def Freeze(self): + self._frozen = True + + def _WillPartOfPathChange(self): + if self._frozen: + raise Exception('The project is frozen. You cannot edit it now') + self._loader = None + + @staticmethod + def FromDict(d): + return Project(d['source_paths']) + + def AsDict(self): + return { + 'source_paths': list(self.source_paths) + } + + def __repr__(self): + return "Project(%s)" % repr(self.source_paths) + + def AddSourcePath(self, path): + self.source_paths.append(path) + + @property + def loader(self): + if self._loader is None: + self._loader = resource_loader.ResourceLoader(self) + return self._loader + + def ResetLoader(self): + self._loader = None + + def _Load(self, filenames): + return [self.loader.LoadModule(module_filename=filename) for + filename in filenames] + + def LoadModule(self, module_name=None, module_filename=None): + return self.loader.LoadModule(module_name=module_name, + module_filename=module_filename) + + def CalcLoadSequenceForModuleNames(self, module_names, + excluded_scripts=None): + modules = [self.loader.LoadModule(module_name=name, + excluded_scripts=excluded_scripts) for + name in module_names] + return self.CalcLoadSequenceForModules(modules) + + def CalcLoadSequenceForModules(self, modules): + already_loaded_set = set() + load_sequence = [] + for m in modules: + m.ComputeLoadSequenceRecursive(load_sequence, already_loaded_set) + return load_sequence + + def GetDepsGraphFromModuleNames(self, module_names): + modules = [self.loader.LoadModule(module_name=name) for + name in module_names] + return self.GetDepsGraphFromModules(modules) + + def GetDepsGraphFromModules(self, modules): + load_sequence = self.CalcLoadSequenceForModules(modules) + g = _Graph() + for m in load_sequence: + g.AddModule(m) + + for dep in m.dependent_modules: + g.AddEdge(m, dep.id) + + # FIXME: _GetGraph is not defined. Maybe `return g` is intended? + return _GetGraph(load_sequence) + + def GetDominatorGraphForModulesNamed(self, module_names, load_sequence): + modules = [self.loader.LoadModule(module_name=name) + for name in module_names] + return self.GetDominatorGraphForModules(modules, load_sequence) + + def GetDominatorGraphForModules(self, start_modules, load_sequence): + modules_by_id = {} + for m in load_sequence: + modules_by_id[m.id] = m + + module_referrers = collections.defaultdict(list) + for m in load_sequence: + for dep in m.dependent_modules: + module_referrers[dep].append(m) + + # Now start at the top module and reverse. + visited = set() + g = _Graph() + + pending = collections.deque() + pending.extend(start_modules) + while len(pending): + cur = pending.pop() + + g.AddModule(cur) + visited.add(cur) + + for out_dep in module_referrers[cur]: + if out_dep in visited: + continue + g.AddEdge(out_dep, cur) + visited.add(out_dep) + pending.append(out_dep) + + # Visited -> Dot + return g.GetDot() + + +class _Graph(object): + + def __init__(self): + self.nodes = [] + self.edges = [] + + def AddModule(self, m): + f = StringIO() + m.AppendJSContentsToFile(f, False, None) + + attrs = { + 'label': '%s (%i)' % (m.name, f.tell()) + } + + f.close() + + attr_items = ['%s="%s"' % (x, y) for x, y in attrs.iteritems()] + node = 'M%i [%s];' % (m.id, ','.join(attr_items)) + self.nodes.append(node) + + def AddEdge(self, mFrom, mTo): + edge = 'M%i -> M%i;' % (mFrom.id, mTo.id) + self.edges.append(edge) + + def GetDot(self): + return 'digraph deps {\n\n%s\n\n%s\n}\n' % ( + '\n'.join(self.nodes), '\n'.join(self.edges)) diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/resource.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/resource.py new file mode 100644 index 00000000000..853dff94437 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/resource.py @@ -0,0 +1,57 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A Resource is a file and its various associated canonical names.""" + +import codecs +import os + + +class Resource(object): + """Represents a file found via a path search.""" + + def __init__(self, toplevel_dir, absolute_path, binary=False): + self.toplevel_dir = toplevel_dir + self.absolute_path = absolute_path + self._contents = None + self._binary = binary + + @property + def relative_path(self): + """The path to the file from the top-level directory""" + return os.path.relpath(self.absolute_path, self.toplevel_dir) + + @property + def unix_style_relative_path(self): + return self.relative_path.replace(os.sep, '/') + + @property + def name(self): + """The dotted name for this resource based on its relative path.""" + return self.name_from_relative_path(self.relative_path) + + @staticmethod + def name_from_relative_path(relative_path): + dirname = os.path.dirname(relative_path) + basename = os.path.basename(relative_path) + modname = os.path.splitext(basename)[0] + if len(dirname): + name = dirname.replace(os.path.sep, '.') + '.' + modname + else: + name = modname + return name + + @property + def contents(self): + if self._contents: + return self._contents + if not os.path.exists(self.absolute_path): + raise Exception('%s not found.' % self.absolute_path) + if self._binary: + f = open(self.absolute_path, mode='rb') + else: + f = codecs.open(self.absolute_path, mode='r', encoding='utf-8') + self._contents = f.read() + f.close() + return self._contents diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/resource_loader.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/resource_loader.py new file mode 100644 index 00000000000..015adaa6608 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/resource_loader.py @@ -0,0 +1,228 @@ +# Copyright (c) 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""ResourceFinder is a helper class for finding resources given their name.""" + +import codecs +import os + +from py_vulcanize import module +from py_vulcanize import style_sheet as style_sheet_module +from py_vulcanize import resource as resource_module +from py_vulcanize import html_module +from py_vulcanize import strip_js_comments + + +class ResourceLoader(object): + """Manges loading modules and their dependencies from files. + + Modules handle parsing and the construction of their individual dependency + pointers. The loader deals with bookkeeping of what has been loaded, and + mapping names to file resources. + """ + + def __init__(self, project): + self.project = project + self.stripped_js_by_filename = {} + self.loaded_modules = {} + self.loaded_raw_scripts = {} + self.loaded_style_sheets = {} + self.loaded_images = {} + + @property + def source_paths(self): + """A list of base directories to search for modules under.""" + return self.project.source_paths + + def FindResource(self, some_path, binary=False): + """Finds a Resource for the given path. + + Args: + some_path: A relative or absolute path to a file. + + Returns: + A Resource or None. + """ + if os.path.isabs(some_path): + return self.FindResourceGivenAbsolutePath(some_path, binary) + else: + return self.FindResourceGivenRelativePath(some_path, binary) + + def FindResourceGivenAbsolutePath(self, absolute_path, binary=False): + """Returns a Resource for the given absolute path.""" + candidate_paths = [] + for source_path in self.source_paths: + if absolute_path.startswith(source_path): + candidate_paths.append(source_path) + if len(candidate_paths) == 0: + return None + + # Sort by length. Longest match wins. + candidate_paths.sort(lambda x, y: len(x) - len(y)) + longest_candidate = candidate_paths[-1] + return resource_module.Resource(longest_candidate, absolute_path, binary) + + def FindResourceGivenRelativePath(self, relative_path, binary=False): + """Returns a Resource for the given relative path.""" + absolute_path = None + for script_path in self.source_paths: + absolute_path = os.path.join(script_path, relative_path) + if os.path.exists(absolute_path): + return resource_module.Resource(script_path, absolute_path, binary) + return None + + def _FindResourceGivenNameAndSuffix( + self, requested_name, extension, return_resource=False): + """Searches for a file and reads its contents. + + Args: + requested_name: The name of the resource that was requested. + extension: The extension for this requested resource. + + Returns: + A (path, contents) pair. + """ + pathy_name = requested_name.replace('.', os.sep) + filename = pathy_name + extension + + resource = self.FindResourceGivenRelativePath(filename) + if return_resource: + return resource + if not resource: + return None, None + return _read_file(resource.absolute_path) + + def FindModuleResource(self, requested_module_name): + """Finds a module javascript file and returns a Resource, or none.""" + js_resource = self._FindResourceGivenNameAndSuffix( + requested_module_name, '.js', return_resource=True) + html_resource = self._FindResourceGivenNameAndSuffix( + requested_module_name, '.html', return_resource=True) + if js_resource and html_resource: + if html_module.IsHTMLResourceTheModuleGivenConflictingResourceNames( + js_resource, html_resource): + return html_resource + return js_resource + elif js_resource: + return js_resource + return html_resource + + def LoadModule(self, module_name=None, module_filename=None, + excluded_scripts=None): + assert bool(module_name) ^ bool(module_filename), ( + 'Must provide either module_name or module_filename.') + if module_filename: + resource = self.FindResource(module_filename) + if not resource: + raise Exception('Could not find %s in %s' % ( + module_filename, repr(self.source_paths))) + module_name = resource.name + else: + resource = None # Will be set if we end up needing to load. + + if module_name in self.loaded_modules: + assert self.loaded_modules[module_name].contents + return self.loaded_modules[module_name] + + if not resource: # happens when module_name was given + resource = self.FindModuleResource(module_name) + if not resource: + raise module.DepsException('No resource for module "%s"' % module_name) + + m = html_module.HTMLModule(self, module_name, resource) + self.loaded_modules[module_name] = m + + # Fake it, this is probably either polymer.min.js or platform.js which are + # actually .js files.... + if resource.absolute_path.endswith('.js'): + return m + + m.Parse(excluded_scripts) + m.Load(excluded_scripts) + return m + + def LoadRawScript(self, relative_raw_script_path): + resource = None + for source_path in self.source_paths: + possible_absolute_path = os.path.join( + source_path, os.path.normpath(relative_raw_script_path)) + if os.path.exists(possible_absolute_path): + resource = resource_module.Resource( + source_path, possible_absolute_path) + break + if not resource: + raise module.DepsException( + 'Could not find a file for raw script %s in %s' % + (relative_raw_script_path, self.source_paths)) + assert relative_raw_script_path == resource.unix_style_relative_path, ( + 'Expected %s == %s' % (relative_raw_script_path, + resource.unix_style_relative_path)) + + if resource.absolute_path in self.loaded_raw_scripts: + return self.loaded_raw_scripts[resource.absolute_path] + + raw_script = module.RawScript(resource) + self.loaded_raw_scripts[resource.absolute_path] = raw_script + return raw_script + + def LoadStyleSheet(self, name): + if name in self.loaded_style_sheets: + return self.loaded_style_sheets[name] + + resource = self._FindResourceGivenNameAndSuffix( + name, '.css', return_resource=True) + if not resource: + raise module.DepsException( + 'Could not find a file for stylesheet %s' % name) + + style_sheet = style_sheet_module.StyleSheet(self, name, resource) + style_sheet.load() + self.loaded_style_sheets[name] = style_sheet + return style_sheet + + def LoadImage(self, abs_path): + if abs_path in self.loaded_images: + return self.loaded_images[abs_path] + + if not os.path.exists(abs_path): + raise module.DepsException("url('%s') did not exist" % abs_path) + + res = self.FindResourceGivenAbsolutePath(abs_path, binary=True) + if res is None: + raise module.DepsException("url('%s') was not in search path" % abs_path) + + image = style_sheet_module.Image(res) + self.loaded_images[abs_path] = image + return image + + def GetStrippedJSForFilename(self, filename, early_out_if_no_py_vulcanize): + if filename in self.stripped_js_by_filename: + return self.stripped_js_by_filename[filename] + + with open(filename, 'r') as f: + contents = f.read(4096) + if early_out_if_no_py_vulcanize and ('py_vulcanize' not in contents): + return None + + s = strip_js_comments.StripJSComments(contents) + self.stripped_js_by_filename[filename] = s + return s + + +def _read_file(absolute_path): + """Reads a file and returns a (path, contents) pair. + + Args: + absolute_path: Absolute path to a file. + + Raises: + Exception: The given file doesn't exist. + IOError: There was a problem opening or reading the file. + """ + if not os.path.exists(absolute_path): + raise Exception('%s not found.' % absolute_path) + f = codecs.open(absolute_path, mode='r', encoding='utf-8') + contents = f.read() + f.close() + return absolute_path, contents diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/resource_unittest.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/resource_unittest.py new file mode 100644 index 00000000000..4da23556f24 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/resource_unittest.py @@ -0,0 +1,17 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import unittest + +from py_vulcanize import resource + + +class ResourceUnittest(unittest.TestCase): + + def testBasic(self): + r = resource.Resource('/a', '/a/b/c.js') + self.assertEquals('b.c', r.name) + self.assertEquals(os.path.join('b', 'c.js'), r.relative_path) + self.assertEquals('b/c.js', r.unix_style_relative_path) diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/strip_js_comments.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/strip_js_comments.py new file mode 100644 index 00000000000..d63c667531a --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/strip_js_comments.py @@ -0,0 +1,81 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Utility function for stripping comments out of JavaScript source code.""" + +import re + + +def _TokenizeJS(text): + """Splits source code text into segments in preparation for comment stripping. + + Note that this doesn't tokenize for parsing. There is no notion of statements, + variables, etc. The only tokens of interest are comment-related tokens. + + Args: + text: The contents of a JavaScript file. + + Yields: + A succession of strings in the file, including all comment-related symbols. + """ + rest = text + tokens = ['//', '/*', '*/', '\n'] + next_tok = re.compile('|'.join(re.escape(x) for x in tokens)) + while len(rest): + m = next_tok.search(rest) + if not m: + # end of string + yield rest + return + min_index = m.start() + end_index = m.end() + + if min_index > 0: + yield rest[:min_index] + + yield rest[min_index:end_index] + rest = rest[end_index:] + + +def StripJSComments(text): + """Strips comments out of JavaScript source code. + + Args: + text: JavaScript source text. + + Returns: + JavaScript source text with comments stripped out. + """ + result_tokens = [] + token_stream = _TokenizeJS(text).__iter__() + while True: + try: + t = token_stream.next() + except StopIteration: + break + + if t == '//': + while True: + try: + t2 = token_stream.next() + if t2 == '\n': + break + except StopIteration: + break + elif t == '/*': + nesting = 1 + while True: + try: + t2 = token_stream.next() + if t2 == '/*': + nesting += 1 + elif t2 == '*/': + nesting -= 1 + if nesting == 0: + break + except StopIteration: + break + else: + result_tokens.append(t) + return ''.join(result_tokens) diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/strip_js_comments_unittest.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/strip_js_comments_unittest.py new file mode 100644 index 00000000000..685cb824a24 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/strip_js_comments_unittest.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for strip_js_comments module.""" + +import unittest + +from py_vulcanize import strip_js_comments + + +# This test case tests a protected method. +# pylint: disable=W0212 +class JavaScriptStripCommentTests(unittest.TestCase): + """Test case for _strip_js_comments and _TokenizeJS.""" + + def test_strip_comments(self): + self.assertEquals( + 'A ', strip_js_comments.StripJSComments('A // foo')) + self.assertEquals( + 'A bar', strip_js_comments.StripJSComments('A // foo\nbar')) + self.assertEquals( + 'A b', strip_js_comments.StripJSComments('A /* foo */ b')) + self.assertEquals( + 'A b', strip_js_comments.StripJSComments('A /* foo\n */ b')) + + def test_tokenize_empty(self): + tokens = list(strip_js_comments._TokenizeJS('')) + self.assertEquals([], tokens) + + def test_tokenize_nl(self): + tokens = list(strip_js_comments._TokenizeJS('\n')) + self.assertEquals(['\n'], tokens) + + def test_tokenize_slashslash_comment(self): + tokens = list(strip_js_comments._TokenizeJS('A // foo')) + self.assertEquals(['A ', '//', ' foo'], tokens) + + def test_tokenize_slashslash_comment_then_newline(self): + tokens = list(strip_js_comments._TokenizeJS('A // foo\nbar')) + self.assertEquals(['A ', '//', ' foo', '\n', 'bar'], tokens) + + def test_tokenize_cstyle_comment_one_line(self): + tokens = list(strip_js_comments._TokenizeJS('A /* foo */')) + self.assertEquals(['A ', '/*', ' foo ', '*/'], tokens) + + def test_tokenize_cstyle_comment_multi_line(self): + tokens = list(strip_js_comments._TokenizeJS('A /* foo\n*bar\n*/')) + self.assertEquals(['A ', '/*', ' foo', '\n', '*bar', '\n', '*/'], tokens) + + +if __name__ == '__main__': + unittest.main() diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/style_sheet.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/style_sheet.py new file mode 100644 index 00000000000..5338762588e --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/style_sheet.py @@ -0,0 +1,138 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import base64 +import os +import re + + +class Image(object): + + def __init__(self, resource): + self.resource = resource + self.aliases = [] + + @property + def relative_path(self): + return self.resource.relative_path + + @property + def absolute_path(self): + return self.resource.absolute_path + + @property + def contents(self): + return self.resource.contents + + +class ParsedStyleSheet(object): + + def __init__(self, loader, containing_dirname, contents): + self.loader = loader + self.contents = contents + self._images = None + self._Load(containing_dirname) + + @property + def images(self): + return self._images + + def AppendDirectlyDependentFilenamesTo(self, dependent_filenames): + for i in self.images: + dependent_filenames.append(i.resource.absolute_path) + + @property + def contents_with_inlined_images(self): + images_by_url = {} + for i in self.images: + for a in i.aliases: + images_by_url[a] = i + + def InlineUrl(m): + url = m.group('url') + image = images_by_url[url] + + ext = os.path.splitext(image.absolute_path)[1] + data = base64.standard_b64encode(image.contents) + + return 'url(data:image/%s;base64,%s)' % (ext[1:], data) + + # I'm assuming we only have url()'s associated with images + return re.sub('url\((?P<quote>"|\'|)(?P<url>[^"\'()]*)(?P=quote)\)', + InlineUrl, self.contents) + + def AppendDirectlyDependentFilenamesTo(self, dependent_filenames): + for i in self.images: + dependent_filenames.append(i.resource.absolute_path) + + def _Load(self, containing_dirname): + if self.contents.find('@import') != -1: + raise Exception('@imports are not supported') + + matches = re.findall( + 'url\((?:["|\']?)([^"\'()]*)(?:["|\']?)\)', + self.contents) + + def resolve_url(url): + if os.path.isabs(url): + # FIXME: module is used here, but py_vulcanize.module is never imported. + # However, py_vulcanize.module cannot be imported since py_vulcanize.module may import + # style_sheet, leading to an import loop. + raise module.DepsException('URL references must be relative') + # URLS are relative to this module's directory + abs_path = os.path.abspath(os.path.join(containing_dirname, url)) + image = self.loader.LoadImage(abs_path) + image.aliases.append(url) + return image + + self._images = [resolve_url(x) for x in matches] + + +class StyleSheet(object): + """Represents a stylesheet resource referenced by a module via the + base.requireStylesheet(xxx) directive.""" + + def __init__(self, loader, name, resource): + self.loader = loader + self.name = name + self.resource = resource + self._parsed_style_sheet = None + + @property + def filename(self): + return self.resource.absolute_path + + @property + def contents(self): + return self.resource.contents + + def __repr__(self): + return 'StyleSheet(%s)' % self.name + + @property + def images(self): + self._InitParsedStyleSheetIfNeeded() + return self._parsed_style_sheet.images + + def AppendDirectlyDependentFilenamesTo(self, dependent_filenames): + self._InitParsedStyleSheetIfNeeded() + + dependent_filenames.append(self.resource.absolute_path) + self._parsed_style_sheet.AppendDirectlyDependentFilenamesTo( + dependent_filenames) + + @property + def contents_with_inlined_images(self): + self._InitParsedStyleSheetIfNeeded() + return self._parsed_style_sheet.contents_with_inlined_images + + def load(self): + self._InitParsedStyleSheetIfNeeded() + + def _InitParsedStyleSheetIfNeeded(self): + if self._parsed_style_sheet: + return + module_dirname = os.path.dirname(self.resource.absolute_path) + self._parsed_style_sheet = ParsedStyleSheet( + self.loader, module_dirname, self.contents) diff --git a/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/style_sheet_unittest.py b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/style_sheet_unittest.py new file mode 100644 index 00000000000..4ebc71d5651 --- /dev/null +++ b/chromium/third_party/catapult/common/py_vulcanize/py_vulcanize/style_sheet_unittest.py @@ -0,0 +1,67 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import base64 +import os +import unittest + +from py_vulcanize import project as project_module +from py_vulcanize import resource_loader +from py_vulcanize import fake_fs +from py_vulcanize import module + + +class StyleSheetUnittest(unittest.TestCase): + + def testImages(self): + fs = fake_fs.FakeFS() + fs.AddFile('/src/foo/x.css', """ +.x .y { + background-image: url(../images/bar.jpeg); +} +""") + fs.AddFile('/src/images/bar.jpeg', 'hello world') + with fs: + project = project_module.Project([os.path.normpath('/src/')]) + loader = resource_loader.ResourceLoader(project) + + foo_x = loader.LoadStyleSheet('foo.x') + self.assertEquals(1, len(foo_x.images)) + + r0 = foo_x.images[0] + self.assertEquals(os.path.normpath('/src/images/bar.jpeg'), + r0.absolute_path) + + inlined = foo_x.contents_with_inlined_images + self.assertEquals(""" +.x .y { + background-image: url(data:image/jpeg;base64,%s); +} +""" % base64.standard_b64encode('hello world'), inlined) + + def testURLResolveFails(self): + fs = fake_fs.FakeFS() + fs.AddFile('/src/foo/x.css', """ +.x .y { + background-image: url(../images/missing.jpeg); +} +""") + with fs: + project = project_module.Project([os.path.normpath('/src')]) + loader = resource_loader.ResourceLoader(project) + + self.assertRaises(module.DepsException, + lambda: loader.LoadStyleSheet('foo.x')) + + def testImportsCauseFailure(self): + fs = fake_fs.FakeFS() + fs.AddFile('/src/foo/x.css', """ +@import url(awesome.css); +""") + with fs: + project = project_module.Project([os.path.normpath('/src')]) + loader = resource_loader.ResourceLoader(project) + + self.assertRaises(Exception, + lambda: loader.LoadStyleSheet('foo.x')) |