diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-01-20 13:40:20 +0100 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-01-22 12:41:23 +0000 |
commit | 7961cea6d1041e3e454dae6a1da660b453efd238 (patch) | |
tree | c0eeb4a9ff9ba32986289c1653d9608e53ccb444 /chromium/build/android | |
parent | b7034d0803538058e5c9d904ef03cf5eab34f6ef (diff) |
BASELINE: Update Chromium to 78.0.3904.130
Change-Id: If185e0c0061b3437531c97c9c8c78f239352a68b
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/build/android')
40 files changed, 781 insertions, 727 deletions
diff --git a/chromium/build/android/BUILD.gn b/chromium/build/android/BUILD.gn index 67fa0f08e17..e2588265ec9 100644 --- a/chromium/build/android/BUILD.gn +++ b/chromium/build/android/BUILD.gn @@ -124,7 +124,7 @@ python_library("resource_sizes_py") { python_library("bundle_wrapper_script_py") { pydeps_file = "gyp/create_bundle_wrapper_script.pydeps" data = [ - "//third_party/android_build_tools/bundletool/bundletool-all-0.10.0.jar", + "//third_party/android_build_tools/bundletool/bundletool-all-0.10.3.jar", ] } diff --git a/chromium/build/android/OWNERS b/chromium/build/android/OWNERS index 2feaebcf833..2330e8d4211 100644 --- a/chromium/build/android/OWNERS +++ b/chromium/build/android/OWNERS @@ -2,6 +2,7 @@ estevenson@chromium.org jbudorick@chromium.org pasko@chromium.org perezju@chromium.org +tiborg@chromium.org wnwen@chromium.org # COMPONENT: Build diff --git a/chromium/build/android/PRESUBMIT.py b/chromium/build/android/PRESUBMIT.py index 0ec045cc23a..32ccfac5047 100644 --- a/chromium/build/android/PRESUBMIT.py +++ b/chromium/build/android/PRESUBMIT.py @@ -37,6 +37,8 @@ def CommonChecks(input_api, output_api): J('..', '..', 'third_party', 'catapult', 'devil'), J('..', '..', 'third_party', 'catapult', 'tracing'), J('..', '..', 'third_party', 'depot_tools'), + J('..', '..', 'third_party', 'colorama', 'src'), + J('..', '..', 'third_party', 'pymock'), ])) output.extend(input_api.canned_checks.RunPylint( input_api, diff --git a/chromium/build/android/apk_operations.py b/chromium/build/android/apk_operations.py index 7395d2c9331..c6efb549f68 100755 --- a/chromium/build/android/apk_operations.py +++ b/chromium/build/android/apk_operations.py @@ -231,7 +231,7 @@ def _InstallBundle(devices, bundle_apks, package_name, command_line_flags_file, # NOTE: For now, installation requires running 'bundletool install-apks'. # TODO(digit): Add proper support for bundles to devil instead, then use it. install_cmd_args = [ - 'install-apks', '--apks=' + bundle_apks, + 'install-apks', '--apks=' + bundle_apks, '--allow-downgrade', '--adb=' + adb_wrapper.AdbWrapper.GetAdbPath(), '--device-id=' + device.serial ] @@ -1525,8 +1525,7 @@ class _BuildBundleApks(_Command): help='Build .apks archive that targets the bundle\'s minSdkVersion and ' 'contains only english splits. It still contains optional splits.') group.add_argument( - '--sdk-version', - help='Implies --minimal. The sdkVersion to build the .apks for.') + '--sdk-version', help='The sdkVersion to build the .apks for.') group.add_argument( '--build-mode', choices=app_bundle_utils.BUILD_APKS_MODES, @@ -1540,7 +1539,7 @@ class _BuildBundleApks(_Command): _GenerateBundleApks( self.bundle_generation_info, self.args.output_apks, - minimal=self.args.sdk_version is not None or self.args.minimal, + minimal=self.args.minimal, minimal_sdk_version=self.args.sdk_version, mode=self.args.build_mode) diff --git a/chromium/build/android/arbitrary_main_dex.flags b/chromium/build/android/arbitrary_main_dex.flags deleted file mode 100644 index 55bfb926219..00000000000 --- a/chromium/build/android/arbitrary_main_dex.flags +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# Proguard flags to pass to R8 which do nothing, and thus allows R8 to -# arbitrarily assign code to different dex files. If we pass a flags file with -# no rules, R8 applies default main dex rules which usually include too much -# and pushes us over the main dex limit. We can't pass higher --min-api since we -# also want to allow multidex in libraries, which are compiled with a low -# min-api. Thus, we have rules which won't do anything to pass as a -# main-dex-rule. - --keep class this.should.not.match.anything { - *; -} diff --git a/chromium/build/android/dcheck_is_off.flags b/chromium/build/android/dcheck_is_off.flags new file mode 100644 index 00000000000..f9059c39d79 --- /dev/null +++ b/chromium/build/android/dcheck_is_off.flags @@ -0,0 +1,12 @@ +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Contains flags that are applied only when ENABLE_DCHECK=false. + +-checkdiscard @org.chromium.base.annotations.CheckDiscard class ** { + *; +} +-checkdiscard class ** { + @org.chromium.base.annotations.CheckDiscard *; +} diff --git a/chromium/build/android/docs/life_of_a_resource.md b/chromium/build/android/docs/life_of_a_resource.md index bd1ffcd994e..1e542c8780e 100644 --- a/chromium/build/android/docs/life_of_a_resource.md +++ b/chromium/build/android/docs/life_of_a_resource.md @@ -183,48 +183,75 @@ none exist, create a new config file and pass its path in your target. The first two bytes of a resource id is the package id. For regular apks, this is `0x7f`. However, Webview is a shared library which gets loaded into other apks. The package id for webview resources is assigned dynamically at runtime. -When webview is loaded it [rewrites all resources][ResourceRewriter.java] to -have the correct package id. When deobfuscating webview resource ids, disregard -the first two bytes in the id when looking it up in the `R.txt` file. +When webview is loaded it calls this [R file's][Base Module R.java File] +onResourcesLoaded function to have the correct package id. When deobfuscating +webview resource ids, disregard the first two bytes in the id when looking it up +in the `R.txt` file. Monochrome, when loaded as webview, rewrites the package ids of resources used by the webview portion to the correct value at runtime, otherwise, its resources have package id `0x7f` when run as a regular apk. -[ResourceRewriter.java]: https://cs.chromium.org/chromium/src/out/android-Debug/gen/android_webview/glue/glue/generated_java/com/android/webview/chromium/ResourceRewriter.java +[Base Module R.java File]: https://cs.chromium.org/chromium/src/out/android-Debug/gen/android_webview/system_webview_apk/generated_java/gen/base_module/R.java ## How R.java files are generated -This is how a sample R.java file looks like: +R.java is a list of static classes, each with multiple static fields containing +ids. These ids are used in java code to reference resources in the apk. + +There are three types of R.java files in Chrome. +1. Base Module Root R.java Files +2. DFM Root R.java Files +3. Source R.java Files +Example Base Module Root R.java File ```java -package org.chromium.ui; +package gen.base_module; public final class R { - public static final class attr { - public static final int buttonAlignment = 0x7f030038; - public static final int buttonColor = 0x7f03003e; - public static final int layout = 0x7f030094; - public static final int roundedfillColor = 0x7f0300bf; - public static final int secondaryButtonText = 0x7f0300c4; - public static final int stackedMargin = 0x7f0300d4; + public static class anim { + public static final int abc_fade_in = 0x7f010000; + public static final int abc_fade_out = 0x7f010001; + public static final int abc_slide_in_top = 0x7f010007; } - public static final class id { - public static final int apart = 0x7f080021; - public static final int dropdown_body_footer_divider = 0x7f08003d; - public static final int dropdown_body_list = 0x7f08003e; - public static final int dropdown_footer = 0x7f08003f; + public static class animator { + public static final int design_appbar_state_list_animator = 0x7f020000; + } +} +``` +Base module root R.java files contain base android resources. All R.java files +can access base module resources through inheritance. + +Example DFM Root R.java File +```java +package gen.vr_module; + +public final class R { + public static class anim extends gen.base_module.R.anim { } - public static final class layout { - public static final int dropdown_item = 0x7f0a0022; - public static final int dropdown_window = 0x7f0a0023; + public static class animator extends gen.base_module.R.animator { + public static final int design_appbar_state_list_animator = 0x7f030000; } } ``` +DFM root R.java files extend base module root R.java files. This allows DFMs to +access their own resources as well as the base module's resources. -R.java is a list of static classes, each with multiple static fields containing -ids. These ids are used in java code to reference resources in the apk. The -R.java file generated via the prepare resources step above has temporary ids +Example Source R.java File +```java +package org.chromium.chrome.vr; + +public final class R { + public static final class anim extends + gen.base_module.R.anim {} + public static final class animator extends + gen.base_module.R.animator {} +} +``` +Source R.java files extend root R.java files and have no resources of their own. +Developers can import these R.java files to access resources in the apk. + +The R.java file generated via the prepare resources step above has temporary ids which are not marked `final`. That R.java file is only used so that javac can compile the java code that references R.*. diff --git a/chromium/build/android/gyp/aar.py b/chromium/build/android/gyp/aar.py index d0f357db334..c31d87b9f11 100755 --- a/chromium/build/android/gyp/aar.py +++ b/chromium/build/android/gyp/aar.py @@ -82,18 +82,22 @@ def _CreateInfo(aar_file): # Some AARs, e.g. gvr_controller_java, have empty R.txt. Such AARs # have no resources as well. We treat empty R.txt as having no R.txt. data['has_r_text_file'] = (z.read('R.txt').strip() != '') + return data - return """\ -# Generated by //build/android/gyp/aar.py -# To regenerate, use "update_android_aar_prebuilts = true" and run "gn gen". -""" + gn_helpers.ToGNString(data) +def _PerformExtract(aar_file, output_dir, name_whitelist): + with build_utils.TempDir() as tmp_dir: + tmp_dir = os.path.join(tmp_dir, 'staging') + os.mkdir(tmp_dir) + build_utils.ExtractAll( + aar_file, path=tmp_dir, predicate=name_whitelist.__contains__) + shutil.rmtree(output_dir, ignore_errors=True) + shutil.move(tmp_dir, output_dir) def _AddCommonArgs(parser): - parser.add_argument('aar_file', - help='Path to the AAR file.', - type=os.path.normpath) + parser.add_argument( + 'aar_file', help='Path to the AAR file.', type=os.path.normpath) def main(): @@ -102,45 +106,53 @@ def main(): subp = command_parsers.add_parser( 'list', help='Output a GN scope describing the contents of the .aar.') _AddCommonArgs(subp) - subp.add_argument('--output', - help='Output file.', - default='-') + subp.add_argument('--output', help='Output file.', default='-') subp = command_parsers.add_parser('extract', help='Extracts the .aar') _AddCommonArgs(subp) - subp.add_argument('--output-dir', - help='Output directory for the extracted files.', - required=True, - type=os.path.normpath) - subp.add_argument('--assert-info-file', - help='Path to .info file. Asserts that it matches what ' - '"list" would output.', - type=argparse.FileType('r')) + subp.add_argument( + '--output-dir', + help='Output directory for the extracted files.', + required=True, + type=os.path.normpath) + subp.add_argument( + '--assert-info-file', + help='Path to .info file. Asserts that it matches what ' + '"list" would output.', + type=argparse.FileType('r')) + subp.add_argument( + '--ignore-resources', + action='store_true', + help='Whether to skip extraction of res/') args = parser.parse_args() + aar_info = _CreateInfo(args.aar_file) + formatted_info = """\ +# Generated by //build/android/gyp/aar.py +# To regenerate, use "update_android_aar_prebuilts = true" and run "gn gen". + +""" + gn_helpers.ToGNString(aar_info) + if args.command == 'extract': if args.assert_info_file: - expected = _CreateInfo(args.aar_file) - actual = args.assert_info_file.read() - if actual != expected: + cached_info = args.assert_info_file.read() + if formatted_info != cached_info: raise Exception('android_aar_prebuilt() cached .info file is ' 'out-of-date. Run gn gen with ' 'update_android_aar_prebuilts=true to update it.') - def clobber(): - # Clear previously extracted versions of the AAR if it is obsolete. - shutil.rmtree(args.output_dir, ignore_errors=True) - build_utils.ExtractAll(args.aar_file, path=args.output_dir) - with zipfile.ZipFile(args.aar_file) as zf: - md5_check.CallAndRecordIfStale( - clobber, input_paths=[args.aar_file], - output_paths=[ - os.path.join(args.output_dir, n) for n in zf.namelist()]) + names = zf.namelist() + if args.ignore_resources: + names = [n for n in names if not n.startswith('res')] + + md5_check.CallAndRecordIfStale( + lambda: _PerformExtract(args.aar_file, args.output_dir, set(names)), + input_paths=[args.aar_file], + output_paths=[os.path.join(args.output_dir, n) for n in names]) elif args.command == 'list': - aar_info = _CreateInfo(args.aar_file) aar_output_present = args.output != '-' and os.path.isfile(args.output) if aar_output_present: # Some .info files are read-only, for examples the cipd-controlled ones @@ -148,14 +160,14 @@ def main(): # that its content is correct, and if it is, exit without touching # the file system. file_info = open(args.output, 'r').read() - if file_info == aar_info: + if file_info == formatted_info: return # Try to write the file. This may fail for read-only ones that were # not updated. try: with open(args.output, 'w') as f: - f.write(aar_info) + f.write(formatted_info) except IOError as e: if not aar_output_present: raise e diff --git a/chromium/build/android/gyp/apkbuilder.py b/chromium/build/android/gyp/apkbuilder.py index 3ec750564f2..a6a5dd95dda 100755 --- a/chromium/build/android/gyp/apkbuilder.py +++ b/chromium/build/android/gyp/apkbuilder.py @@ -203,20 +203,23 @@ def _CreateAssetsList(path_tuples): def _AddNativeLibraries(out_apk, native_libs, android_abi, uncompress): """Add native libraries to APK.""" - has_crazy_linker = any('android_linker' in os.path.basename(p) - for p in native_libs) + has_crazy_linker = any( + 'android_linker' in os.path.basename(p) for p in native_libs) + has_monochrome = any('monochrome' in os.path.basename(p) for p in native_libs) + for path in native_libs: basename = os.path.basename(path) - compress = None - if (uncompress and os.path.splitext(basename)[1] == '.so' - and 'android_linker' not in basename - and (not has_crazy_linker or 'clang_rt' not in basename) - and (not has_crazy_linker or 'crashpad_handler' not in basename)): - compress = False - # Add prefix to prevent android install from extracting upon install. - if has_crazy_linker: - basename = 'crazy.' + basename + if uncompress and os.path.splitext(basename)[1] == '.so': + # Trichrome + if has_crazy_linker and has_monochrome: + compress = False + elif ('android_linker' not in basename + and (not has_crazy_linker or 'clang_rt' not in basename) + and (not has_crazy_linker or 'crashpad_handler' not in basename)): + compress = False + if has_crazy_linker and not has_monochrome: + basename = 'crazy.' + basename apk_path = 'lib/%s/%s' % (android_abi, basename) build_utils.AddToZipHermetic(out_apk, @@ -346,7 +349,7 @@ def main(args): build_utils.AddToZipHermetic(out_apk, apk_path, data='') # 5. Resources - for info in resource_infos: + for info in sorted(resource_infos, key=lambda i: i.filename): if info.filename != 'AndroidManifest.xml': copy_resource(info) @@ -355,7 +358,7 @@ def main(args): # Prebuilt jars may contain class files which we shouldn't include. for java_resource in options.java_resources: with zipfile.ZipFile(java_resource, 'r') as java_resource_jar: - for apk_path in java_resource_jar.namelist(): + for apk_path in sorted(java_resource_jar.namelist()): apk_path_lower = apk_path.lower() if apk_path_lower.startswith('meta-inf/'): diff --git a/chromium/build/android/gyp/assert_static_initializers.py b/chromium/build/android/gyp/assert_static_initializers.py index a03ca1c7b1e..392ee079004 100755 --- a/chromium/build/android/gyp/assert_static_initializers.py +++ b/chromium/build/android/gyp/assert_static_initializers.py @@ -60,17 +60,23 @@ def _PrintDumpSIsCount(apk_so_name, unzipped_so, out_dir, tool_prefix): # Mostly copied from //infra/scripts/legacy/scripts/slave/chromium/sizes.py. -def _ReadInitArray(so_path, tool_prefix): +def _ReadInitArray(so_path, tool_prefix, expect_no_initializers): stdout = _RunReadelf(so_path, ['-SW'], tool_prefix) - # Matches: .ctors PROGBITS 000000000516add0 5169dd0 000010 00 WA 0 0 8 + # Matches: .init_array INIT_ARRAY 000000000516add0 5169dd0 000010 00 WA 0 0 8 match = re.search(r'\.init_array.*$', stdout, re.MULTILINE) - if not match: + if expect_no_initializers: + if match: + raise Exception( + 'Expected no initializers for %s, yet some were found' % so_path) + else: + return 0 + elif not match: raise Exception('Did not find section: .init_array in:\n' + stdout) size_str = re.split(r'\W+', match.group(0))[5] return int(size_str, 16) -def _CountStaticInitializers(so_path, tool_prefix): +def _CountStaticInitializers(so_path, tool_prefix, expect_no_initializers): # Find the number of files with at least one static initializer. # First determine if we're 32 or 64 bit stdout = _RunReadelf(so_path, ['-h'], tool_prefix) @@ -84,12 +90,12 @@ def _CountStaticInitializers(so_path, tool_prefix): # Then find the number of files with global static initializers. # NOTE: this is very implementation-specific and makes assumptions # about how compiler and linker implement global static initializers. - init_array_size = _ReadInitArray(so_path, tool_prefix) + init_array_size = _ReadInitArray(so_path, tool_prefix, expect_no_initializers) return init_array_size / word_size def _AnalyzeStaticInitializers(apk_filename, tool_prefix, dump_sis, out_dir, - ignored_libs): + ignored_libs, no_initializers_libs): # Static initializer counting mostly copies logic in # infra/scripts/legacy/scripts/slave/chromium/sizes.py. with zipfile.ZipFile(apk_filename) as z: @@ -105,10 +111,13 @@ def _AnalyzeStaticInitializers(apk_filename, tool_prefix, dump_sis, out_dir, si_count = 0 for f in files_to_check: + expect_no_initializers = (os.path.basename( + f.filename) in no_initializers_libs) with tempfile.NamedTemporaryFile() as temp: temp.write(z.read(f)) temp.flush() - si_count += _CountStaticInitializers(temp.name, tool_prefix) + si_count += _CountStaticInitializers(temp.name, tool_prefix, + expect_no_initializers) if dump_sis: # Print count and list of SIs reported by dump-static-initializers.py. # Doesn't work well on all archs (particularly arm), which is why @@ -128,11 +137,14 @@ def main(): parser.add_argument('apk', help='APK file path.') args = parser.parse_args() - #TODO(crbug.com/838414): add support for files included via loadable_modules. + # TODO(crbug.com/838414): add support for files included via loadable_modules. ignored_libs = ['libarcore_sdk_c.so'] + # The chromium linker doesn't have static initializers, which makes the + # regular check throw. It should not have any. + no_initializers_libs = ['libchromium_android_linker.so'] si_count = _AnalyzeStaticInitializers(args.apk, args.tool_prefix, False, '.', - ignored_libs) + ignored_libs, no_initializers_libs) if si_count != args.expected_count: print('Expected {} static initializers, but found {}.'.format( args.expected_count, si_count)) @@ -144,7 +156,7 @@ def main(): print('Dumping static initializers via dump-static-initializers.py:') sys.stdout.flush() _AnalyzeStaticInitializers(args.apk, args.tool_prefix, True, '.', - ignored_libs) + ignored_libs, no_initializers_libs) print() print('If the above list is not useful, consider listing them with:') print(' //tools/binary_size/diagnose_bloat.py') diff --git a/chromium/build/android/gyp/bundletool.py b/chromium/build/android/gyp/bundletool.py index 4e06017ef39..2201cc12373 100755 --- a/chromium/build/android/gyp/bundletool.py +++ b/chromium/build/android/gyp/bundletool.py @@ -18,7 +18,7 @@ BUNDLETOOL_DIR = os.path.abspath(os.path.join( __file__, '..', '..', '..', '..', 'third_party', 'android_build_tools', 'bundletool')) -BUNDLETOOL_VERSION = '0.10.0' +BUNDLETOOL_VERSION = '0.10.3' BUNDLETOOL_JAR_PATH = os.path.join( BUNDLETOOL_DIR, 'bundletool-all-%s.jar' % BUNDLETOOL_VERSION) diff --git a/chromium/build/android/gyp/compile_resources.py b/chromium/build/android/gyp/compile_resources.py index 199d8bcbab3..95b8c2aa0d7 100755 --- a/chromium/build/android/gyp/compile_resources.py +++ b/chromium/build/android/gyp/compile_resources.py @@ -516,16 +516,11 @@ https://chromium.googlesource.com/chromium/src/+/HEAD/chrome/android/java/README sys.exit(1) -def _ResourceNameFromPath(path): - return os.path.splitext(os.path.basename(path))[0] - - -def _CreateKeepPredicate(resource_dirs, resource_blacklist_regex, +def _CreateKeepPredicate(resource_blacklist_regex, resource_blacklist_exceptions): """Return a predicate lambda to determine which resource files to keep. Args: - resource_dirs: list of top-level resource directories. resource_blacklist_regex: A regular expression describing all resources to exclude, except if they are mip-maps, or if they are listed in |resource_blacklist_exceptions|. @@ -535,35 +530,16 @@ def _CreateKeepPredicate(resource_dirs, resource_blacklist_regex, A lambda that takes a path, and returns true if the corresponding file must be kept. """ - naive_predicate = lambda path: os.path.basename(path)[0] != '.' + predicate = lambda path: os.path.basename(path)[0] != '.' if resource_blacklist_regex == '': # Do not extract dotfiles (e.g. ".gitkeep"). aapt ignores them anyways. - return naive_predicate - - if resource_blacklist_regex != '': - # A simple predicate that only removes (returns False for) paths covered by - # the blacklist regex, except if they are mipmaps, or listed as exceptions. - naive_predicate = lambda path: ( - not re.search(resource_blacklist_regex, path) or - re.search(r'[/-]mipmap[/-]', path) or - build_utils.MatchesGlob(path, resource_blacklist_exceptions)) - - # Build a set of all names from drawables kept by naive_predicate(). - # Used later to ensure that we never exclude drawables from densities - # that are filtered-out by naive_predicate(). - non_filtered_drawables = set() - for resource_dir in resource_dirs: - for path in _IterFiles(resource_dir): - if re.search(r'[/-]drawable[/-]', path) and naive_predicate(path): - non_filtered_drawables.add(_ResourceNameFromPath(path)) - - # NOTE: Defined as a function, instead of a lambda to avoid the - # auto-formatter to put this on a very long line that overflows. - def drawable_predicate(path): - return (naive_predicate(path) - or _ResourceNameFromPath(path) not in non_filtered_drawables) + return predicate - return drawable_predicate + # A simple predicate that only removes (returns False for) paths covered by + # the blacklist regex or listed as exceptions. + return lambda path: ( + not re.search(resource_blacklist_regex, path) or + build_utils.MatchesGlob(path, resource_blacklist_exceptions)) def _ConvertToWebP(webp_binary, png_files): @@ -720,8 +696,7 @@ def _PackageApk(options, build): # Create a function that selects which resource files should be packaged # into the final output. Any file that does not pass the predicate will # be removed below. - keep_predicate = _CreateKeepPredicate(dep_subdirs, - options.resource_blacklist_regex, + keep_predicate = _CreateKeepPredicate(options.resource_blacklist_regex, options.resource_blacklist_exceptions) png_paths = [] for directory in dep_subdirs: diff --git a/chromium/build/android/gyp/create_app_bundle_minimal_apks.py b/chromium/build/android/gyp/create_app_bundle_apks.py index f01691e418f..d261996de35 100755 --- a/chromium/build/android/gyp/create_app_bundle_minimal_apks.py +++ b/chromium/build/android/gyp/create_app_bundle_apks.py @@ -4,7 +4,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Creates an .apks from an .aab with only English strings.""" +"""Creates an .apks from an .aab.""" import argparse import os @@ -28,6 +28,10 @@ def main(): '--keystore-password', required=True, help='Keystore password.') parser.add_argument( '--keystore-name', required=True, help='Key name within keystore') + parser.add_argument( + '--minimal', + action='store_true', + help='Create APKs archive with minimal language support.') args = parser.parse_args() @@ -38,7 +42,7 @@ def main(): args.keystore_path, args.keystore_password, args.keystore_name, - minimal=True, + minimal=args.minimal, check_for_noop=False) diff --git a/chromium/build/android/gyp/create_app_bundle_minimal_apks.pydeps b/chromium/build/android/gyp/create_app_bundle_apks.pydeps index cd5b08158fb..bdee0af2c2d 100644 --- a/chromium/build/android/gyp/create_app_bundle_minimal_apks.pydeps +++ b/chromium/build/android/gyp/create_app_bundle_apks.pydeps @@ -1,5 +1,5 @@ # Generated by running: -# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_app_bundle_minimal_apks.pydeps build/android/gyp/create_app_bundle_minimal_apks.py +# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_app_bundle_apks.pydeps build/android/gyp/create_app_bundle_apks.py ../../../third_party/jinja2/__init__.py ../../../third_party/jinja2/_compat.py ../../../third_party/jinja2/bccache.py @@ -26,7 +26,7 @@ ../pylib/utils/__init__.py ../pylib/utils/app_bundle_utils.py bundletool.py -create_app_bundle_minimal_apks.py +create_app_bundle_apks.py util/__init__.py util/build_utils.py util/md5_check.py diff --git a/chromium/build/android/gyp/javac.py b/chromium/build/android/gyp/javac.py index f34415de7fd..e8f27d562c2 100755 --- a/chromium/build/android/gyp/javac.py +++ b/chromium/build/android/gyp/javac.py @@ -245,8 +245,13 @@ def _ProcessInfo(java_file, package_name, class_names, source, chromium_code): _CheckPathMatchesClassName(java_file, package_name, class_names[0]) +def _ShouldIncludeInJarInfo(fully_qualified_name, excluded_globs): + name_as_class_glob = fully_qualified_name.replace('.', '/') + '.class' + return not build_utils.MatchesGlob(name_as_class_glob, excluded_globs) + + def _CreateInfoFile(java_files, jar_path, chromium_code, srcjar_files, - classes_dir, generated_java_dir): + classes_dir, generated_java_dir, excluded_globs): """Writes a .jar.info file. This maps fully qualified names for classes to either the java file that they @@ -269,7 +274,8 @@ def _CreateInfoFile(java_files, jar_path, chromium_code, srcjar_files, source = srcjar_files.get(java_file, java_file) for fully_qualified_name in _ProcessInfo( java_file, package_name, class_names, source, chromium_code): - all_info_data[fully_qualified_name] = java_file + if _ShouldIncludeInJarInfo(fully_qualified_name, excluded_globs): + all_info_data[fully_qualified_name] = java_file logging.info('Writing info file: %s', output_path) with build_utils.AtomicOutput(output_path) as f: jar_info_utils.WriteJarInfoFile(f, all_info_data, srcjar_files) @@ -364,7 +370,8 @@ def _OnStaleMd5(options, javac_cmd, java_files, classpath): if save_outputs: _CreateInfoFile(java_files, options.jar_path, options.chromium_code, - srcjar_files, classes_dir, generated_java_dir) + srcjar_files, classes_dir, generated_java_dir, + options.jar_info_exclude_globs) else: build_utils.Touch(options.jar_path + '.info') @@ -431,6 +438,9 @@ def _ParseOptions(argv): 'files are packaged into the jar. Files should be specified in ' 'format <filename>:<path to be placed in jar>.') parser.add_option( + '--jar-info-exclude-globs', + help='GN list of exclude globs to filter from generated .info files.') + parser.add_option( '--chromium-code', type='int', help='Whether code being compiled should be built with stricter ' @@ -458,6 +468,8 @@ def _ParseOptions(argv): options.processorpath = build_utils.ParseGnList(options.processorpath) options.processors = build_utils.ParseGnList(options.processors) options.java_srcjars = build_utils.ParseGnList(options.java_srcjars) + options.jar_info_exclude_globs = build_utils.ParseGnList( + options.jar_info_exclude_globs) if options.java_version == '1.8' and options.bootclasspath: # Android's boot jar doesn't contain all java 8 classes. @@ -579,14 +591,15 @@ def main(argv): options.jar_path + '.info', ] - # List python deps in input_strings rather than input_paths since the contents - # of them does not change what gets written to the depsfile. + input_strings = javac_cmd + classpath + if options.jar_info_exclude_globs: + input_strings.append(options.jar_info_exclude_globs) build_utils.CallAndWriteDepfileIfStale( lambda: _OnStaleMd5(options, javac_cmd, java_files, classpath), options, depfile_deps=depfile_deps, input_paths=input_paths, - input_strings=javac_cmd + classpath, + input_strings=input_strings, output_paths=output_paths, add_pydeps=False) logging.info('Script complete: %s', __file__) diff --git a/chromium/build/android/gyp/lint.py b/chromium/build/android/gyp/lint.py index ac645d7fd83..605a478ec85 100755 --- a/chromium/build/android/gyp/lint.py +++ b/chromium/build/android/gyp/lint.py @@ -165,7 +165,15 @@ def _OnStaleMd5(lint_path, src_dir = _NewTempSubdir('SRC_ROOT') src_dirs.append(src_dir) cmd.extend(['--sources', _RebasePath(src_dir)]) - os.symlink(os.path.abspath(src), PathInDir(src_dir, src)) + # In cases where the build dir is outside of the src dir, this can + # result in trying to symlink a file to itself for this file: + # gen/components/version_info/android/java/org/chromium/ + # components/version_info/VersionConstants.java + src = os.path.abspath(src) + dst = PathInDir(src_dir, src) + if src == dst: + continue + os.symlink(src, dst) if srcjars: srcjar_paths = build_utils.ParseGnList(srcjars) diff --git a/chromium/build/android/gyp/main_dex_list.py b/chromium/build/android/gyp/main_dex_list.py index 520be39d9c0..d610982699c 100755 --- a/chromium/build/android/gyp/main_dex_list.py +++ b/chromium/build/android/gyp/main_dex_list.py @@ -12,7 +12,6 @@ import tempfile import zipfile from util import build_utils -from util import proguard_util def main(args): diff --git a/chromium/build/android/gyp/main_dex_list.pydeps b/chromium/build/android/gyp/main_dex_list.pydeps index 8c482dfa523..1d562703def 100644 --- a/chromium/build/android/gyp/main_dex_list.pydeps +++ b/chromium/build/android/gyp/main_dex_list.pydeps @@ -5,4 +5,3 @@ main_dex_list.py util/__init__.py util/build_utils.py util/md5_check.py -util/proguard_util.py diff --git a/chromium/build/android/gyp/native_libraries_template.py b/chromium/build/android/gyp/native_libraries_template.py index 901fd86542f..0a70d95b082 100644 --- a/chromium/build/android/gyp/native_libraries_template.py +++ b/chromium/build/android/gyp/native_libraries_template.py @@ -19,12 +19,13 @@ public class NativeLibraries {{ public static {MAYBE_FINAL}boolean sUseLinker{USE_LINKER}; public static {MAYBE_FINAL}boolean sUseLibraryInZipFile{USE_LIBRARY_IN_ZIP_FILE}; public static {MAYBE_FINAL}boolean sEnableLinkerTests{ENABLE_LINKER_TESTS}; + public static {MAYBE_FINAL}boolean sUseModernLinker{USE_MODERN_LINKER}; // This is the list of native libraries to be loaded (in the correct order) // by LibraryLoader.java. // TODO(cjhopman): This is public since it is referenced by NativeTestActivity.java // directly. The two ways of library loading should be refactored into one. - public static {MAYBE_FINAL}String[] LIBRARIES = {LIBRARIES}; + public static {MAYBE_FINAL}String[] LIBRARIES = {{{LIBRARIES}}}; // This is the expected version of the 'main' native library, which is the one that // implements the initial set of base JNI functions including diff --git a/chromium/build/android/gyp/proguard.py b/chromium/build/android/gyp/proguard.py index f0f7a30d0dc..3284341100d 100755 --- a/chromium/build/android/gyp/proguard.py +++ b/chromium/build/android/gyp/proguard.py @@ -4,114 +4,128 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import cStringIO -import optparse +import argparse import os import re import shutil import sys -import tempfile +import zipfile from util import build_utils from util import diff_utils -from util import proguard_util - -_GENERATED_PROGUARD_HEADER = """ -################################################################################ -# Dynamically generated from build/android/gyp/proguard.py -################################################################################ -""" - -# Example: -# android.arch.core.internal.SafeIterableMap$Entry -> b: -# 1:1:java.lang.Object getKey():353:353 -> getKey -# 2:2:java.lang.Object getValue():359:359 -> getValue -def _RemoveMethodMappings(orig_path, out_fd): - with open(orig_path) as in_fd: - for line in in_fd: - if line[:1] != ' ': - out_fd.write(line) - out_fd.flush() - - -def _ParseOptions(args): - parser = optparse.OptionParser() + + +class _ProguardOutputFilter(object): + """ProGuard outputs boring stuff to stdout (ProGuard version, jar path, etc) + as well as interesting stuff (notes, warnings, etc). If stdout is entirely + boring, this class suppresses the output. + """ + + IGNORE_RE = re.compile( + r'Pro.*version|Note:|Reading|Preparing|Printing|ProgramClass:|Searching|' + r'jar \[|\d+ class path entries checked') + + def __init__(self): + self._last_line_ignored = False + self._ignore_next_line = False + + def __call__(self, output): + ret = [] + for line in output.splitlines(True): + if self._ignore_next_line: + self._ignore_next_line = False + continue + + if '***BINARY RUN STATS***' in line: + self._last_line_ignored = True + self._ignore_next_line = True + elif not line.startswith(' '): + self._last_line_ignored = bool(self.IGNORE_RE.match(line)) + elif 'You should check if you need to specify' in line: + self._last_line_ignored = True + + if not self._last_line_ignored: + ret.append(line) + return ''.join(ret) + + +def _ParseOptions(): + args = build_utils.ExpandFileArgs(sys.argv[1:]) + parser = argparse.ArgumentParser() build_utils.AddDepfileOption(parser) - parser.add_option('--proguard-path', - help='Path to the proguard.jar to use.') - parser.add_option('--r8-path', - help='Path to the R8.jar to use.') - parser.add_option('--input-paths', - help='Paths to the .jar files proguard should run on.') - parser.add_option('--output-path', help='Path to the generated .jar file.') - parser.add_option('--proguard-configs', action='append', - help='Paths to proguard configuration files.') - parser.add_option('--proguard-config-exclusions', - default='', - help='GN list of paths to proguard configuration files ' - 'included by --proguard-configs, but that should ' - 'not actually be included.') - parser.add_option( - '--apply-mapping', help='Path to proguard mapping to apply.') - parser.add_option('--mapping-output', - help='Path for proguard to output mapping file to.') - parser.add_option( + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--proguard-path', help='Path to the proguard.jar to use.') + group.add_argument('--r8-path', help='Path to the R8.jar to use.') + parser.add_argument( + '--input-paths', required=True, help='GN-list of .jar files to optimize.') + parser.add_argument( + '--output-path', required=True, help='Path to the generated .jar file.') + parser.add_argument( + '--proguard-configs', + action='append', + required=True, + help='GN-list of configuration files.') + parser.add_argument( + '--apply-mapping', help='Path to ProGuard mapping to apply.') + parser.add_argument( + '--mapping-output', + required=True, + help='Path for ProGuard to output mapping file to.') + parser.add_argument( '--extra-mapping-output-paths', - help='Additional paths to copy output mapping file to.') - parser.add_option( + help='GN-list of additional paths to copy output mapping file to.') + parser.add_argument( '--output-config', - help='Path to write the merged proguard config file to.') - parser.add_option( + help='Path to write the merged ProGuard config file to.') + parser.add_argument( '--expected-configs-file', - help='Path to a file containing the expected merged proguard configs') - parser.add_option( + help='Path to a file containing the expected merged ProGuard configs') + parser.add_argument( '--verify-expected-configs', action='store_true', - help='Fail if the expected merged proguard configs differ from the ' - 'generated merged proguard configs.') - parser.add_option('--classpath', action='append', - help='Classpath for proguard.') - parser.add_option('--main-dex-rules-path', action='append', - help='Paths to main dex rules for multidex' - '- only works with R8.') - parser.add_option('--min-api', default='', - help='Minimum Android API level compatibility.') - parser.add_option('--verbose', '-v', action='store_true', - help='Print all proguard output') - parser.add_option( + help='Fail if the expected merged ProGuard configs differ from the ' + 'generated merged ProGuard configs.') + parser.add_argument( + '--classpath', + action='append', + help='GN-list of .jar files to include as libraries.') + parser.add_argument( + '--main-dex-rules-path', + action='append', + help='Path to main dex rules for multidex' + '- only works with R8.') + parser.add_argument( + '--min-api', help='Minimum Android API level compatibility.') + parser.add_argument( + '--verbose', '-v', action='store_true', help='Print all ProGuard output') + parser.add_argument( '--repackage-classes', help='Unique package name given to an asynchronously proguarded module') - parser.add_option( + parser.add_argument( '--disable-outlining', action='store_true', help='Disable the outlining optimization provided by R8.') - options, _ = parser.parse_args(args) + options = parser.parse_args(args) - assert not options.main_dex_rules_path or options.r8_path, \ - 'R8 must be enabled to pass main dex rules.' + if options.main_dex_rules_path and not options.r8_path: + parser.error('R8 must be enabled to pass main dex rules.') - classpath = [] - for arg in options.classpath: - classpath += build_utils.ParseGnList(arg) - options.classpath = classpath + if options.expected_configs_file and not options.output_config: + parser.error('--expected-configs-file requires --output-config') - configs = [] - for arg in options.proguard_configs: - configs += build_utils.ParseGnList(arg) - options.proguard_configs = configs - options.proguard_config_exclusions = ( - build_utils.ParseGnList(options.proguard_config_exclusions)) + if options.proguard_path and options.disable_outlining: + parser.error('--disable-outlining requires --r8-path') + options.classpath = build_utils.ParseGnList(options.classpath) + options.proguard_configs = build_utils.ParseGnList(options.proguard_configs) options.input_paths = build_utils.ParseGnList(options.input_paths) - - if not options.mapping_output: - options.mapping_output = options.output_path + '.mapping' + options.extra_mapping_output_paths = build_utils.ParseGnList( + options.extra_mapping_output_paths) if options.apply_mapping: options.apply_mapping = os.path.abspath(options.apply_mapping) - return options @@ -121,7 +135,7 @@ def _VerifyExpectedConfigs(expected_path, actual_path, fail_on_exit): return sys.stderr.write("""\ -Proguard flag expectations file needs updating. For details see: +ProGuard flag expectations file needs updating. For details see: https://chromium.googlesource.com/chromium/src/+/HEAD/chrome/android/java/README.md """) sys.stderr.write(msg) @@ -129,68 +143,206 @@ https://chromium.googlesource.com/chromium/src/+/HEAD/chrome/android/java/README sys.exit(1) -def _MoveTempDexFile(tmp_dex_dir, dex_path): - """Move the temp dex file out of |tmp_dex_dir|. +def _OptimizeWithR8(options, + config_paths, + libraries, + dynamic_config_data, + print_stdout=False): + with build_utils.TempDir() as tmp_dir: + if dynamic_config_data: + tmp_config_path = os.path.join(tmp_dir, 'proguard_config.txt') + with open(tmp_config_path, 'w') as f: + f.write(dynamic_config_data) + config_paths = config_paths + [tmp_config_path] + + tmp_mapping_path = os.path.join(tmp_dir, 'mapping.txt') + # If there is no output (no classes are kept), this prevents this script + # from failing. + build_utils.Touch(tmp_mapping_path) + + output_is_zipped = not options.output_path.endswith('.dex') + tmp_output = os.path.join(tmp_dir, 'r8out') + if output_is_zipped: + tmp_output += '.jar' + else: + os.mkdir(tmp_output) + + cmd = [ + 'java', + '-jar', + options.r8_path, + '--no-desugaring', + '--no-data-resources', + '--output', + tmp_output, + '--pg-map-output', + tmp_mapping_path, + ] + + for lib in libraries: + cmd += ['--lib', lib] + + for config_file in config_paths: + cmd += ['--pg-conf', config_file] + + if options.min_api: + cmd += ['--min-api', options.min_api] + + if options.main_dex_rules_path: + for main_dex_rule in options.main_dex_rules_path: + cmd += ['--main-dex-rules', main_dex_rule] + + cmd += options.input_paths + + stderr_filter = None + env = os.environ.copy() + if options.disable_outlining: + stderr_filter = lambda l: re.sub(r'.*_JAVA_OPTIONS.*\n?', '', l) + env['_JAVA_OPTIONS'] = '-Dcom.android.tools.r8.disableOutlining=1' + + build_utils.CheckOutput( + cmd, env=env, print_stdout=print_stdout, stderr_filter=stderr_filter) + + if not output_is_zipped: + found_files = os.listdir(tmp_output) + if len(found_files) > 1: + raise Exception('Too many files created: {}'.format(found_files)) + tmp_output = os.path.join(tmp_output, found_files[0]) + + # Copy output files to correct locations. + shutil.move(tmp_output, options.output_path) + + with open(options.mapping_output, 'w') as out_file, \ + open(tmp_mapping_path) as in_file: + # Mapping files generated by R8 include comments that may break + # some of our tooling so remove those (specifically: apkanalyzer). + out_file.writelines(l for l in in_file if not l.startswith('#')) + + +def _OptimizeWithProguard(options, + config_paths, + libraries, + dynamic_config_data, + print_stdout=False): + with build_utils.TempDir() as tmp_dir: + combined_injars_path = os.path.join(tmp_dir, 'injars.jar') + combined_libjars_path = os.path.join(tmp_dir, 'libjars.jar') + combined_proguard_configs_path = os.path.join(tmp_dir, 'includes.txt') + tmp_mapping_path = os.path.join(tmp_dir, 'mapping.txt') + tmp_output_jar = os.path.join(tmp_dir, 'output.jar') + + build_utils.MergeZips(combined_injars_path, options.input_paths) + build_utils.MergeZips(combined_libjars_path, libraries) + with open(combined_proguard_configs_path, 'w') as f: + f.write(_CombineConfigs(config_paths, dynamic_config_data)) + + if options.proguard_path.endswith('.jar'): + cmd = [ + 'java', '-jar', options.proguard_path, '-include', + combined_proguard_configs_path + ] + else: + cmd = [options.proguard_path, '@' + combined_proguard_configs_path] + + cmd += [ + '-forceprocessing', + '-libraryjars', + combined_libjars_path, + '-injars', + combined_injars_path, + '-outjars', + tmp_output_jar, + '-printmapping', + tmp_mapping_path, + ] + + # Warning: and Error: are sent to stderr, but messages and Note: are sent + # to stdout. + stdout_filter = None + stderr_filter = None + if print_stdout: + stdout_filter = _ProguardOutputFilter() + stderr_filter = _ProguardOutputFilter() + build_utils.CheckOutput( + cmd, + print_stdout=True, + print_stderr=True, + stdout_filter=stdout_filter, + stderr_filter=stderr_filter) + + # ProGuard will skip writing if the file would be empty. + build_utils.Touch(tmp_mapping_path) + + # Copy output files to correct locations. + shutil.move(tmp_output_jar, options.output_path) + shutil.move(tmp_mapping_path, options.mapping_output) + + +def _CombineConfigs(configs, dynamic_config_data, exclude_generated=False): + ret = [] + + def add_header(name): + ret.append('#' * 80) + ret.append('# ' + name) + ret.append('#' * 80) + + for config in sorted(configs): + if exclude_generated and config.endswith('.resources.proguard.txt'): + continue + + add_header(config) + with open(config) as config_file: + contents = config_file.read().rstrip() + # Fix up line endings (third_party configs can have windows endings). + contents = contents.replace('\r', '') + # Remove numbers from generated rule comments to make file more + # diff'able. + contents = re.sub(r' #generated:\d+', '', contents) + ret.append(contents) + ret.append('') + + if dynamic_config_data: + add_header('Dynamically generated from build/android/gyp/proguard.py') + ret.append(dynamic_config_data) + ret.append('') + return '\n'.join(ret) + + +def _CreateDynamicConfig(options): + ret = [] + if not options.r8_path and options.min_api: + # R8 adds this option automatically, and uses -assumenosideeffects instead + # (which ProGuard doesn't support doing). + ret.append("""\ +-assumevalues class android.os.Build$VERSION { + public static final int SDK_INT return %s..9999; +}""" % options.min_api) - Args: - tmp_dex_dir: Path to temporary directory created with tempfile.mkdtemp(). - The directory should have just a single file. - dex_path: Target path to move dex file to. + if options.apply_mapping: + ret.append("-applymapping '%s'" % options.apply_mapping) + if options.repackage_classes: + ret.append("-repackageclasses '%s'" % options.repackage_classes) + return '\n'.join(ret) + + +def _VerifyNoEmbeddedConfigs(jar_paths): + failed = False + for jar_path in jar_paths: + with zipfile.ZipFile(jar_path) as z: + for name in z.namelist(): + if name.startswith('META-INF/proguard/'): + failed = True + sys.stderr.write("""\ +Found embedded proguard config within {}. +Embedded configs are not permitted (https://crbug.com/989505) +""".format(jar_path)) + break + if failed: + sys.exit(1) - Raises: - Exception if there are multiple files in |tmp_dex_dir|. - """ - tempfiles = os.listdir(tmp_dex_dir) - if len(tempfiles) > 1: - raise Exception('%d files created, expected 1' % len(tempfiles)) - - tmp_dex_path = os.path.join(tmp_dex_dir, tempfiles[0]) - shutil.move(tmp_dex_path, dex_path) - - -def _CreateR8Command(options, map_output_path, output_dir, tmp_config_path, - libraries): - cmd = [ - 'java', '-jar', options.r8_path, - '--no-desugaring', - '--no-data-resources', - '--output', output_dir, - '--pg-map-output', map_output_path, - ] - - for lib in libraries: - cmd += ['--lib', lib] - - for config_file in options.proguard_configs: - cmd += ['--pg-conf', config_file] - - temp_config_string = '' - if options.apply_mapping or options.repackage_classes or options.min_api: - with open(tmp_config_path, 'w') as f: - if options.apply_mapping: - temp_config_string += '-applymapping \'%s\'\n' % (options.apply_mapping) - if options.repackage_classes: - temp_config_string += '-repackageclasses \'%s\'\n' % ( - options.repackage_classes) - if options.min_api: - temp_config_string += ( - '-assumenosideeffects class android.os.Build$VERSION {\n' + - ' public static final int SDK_INT return ' + options.min_api + - '..9999;\n}\n') - f.write(temp_config_string) - cmd += ['--pg-conf', tmp_config_path] - - if options.main_dex_rules_path: - for main_dex_rule in options.main_dex_rules_path: - cmd += ['--main-dex-rules', main_dex_rule] - - cmd += options.input_paths - return cmd, temp_config_string - - -def main(args): - args = build_utils.ExpandFileArgs(args) - options = _ParseOptions(args) + +def main(): + options = _ParseOptions() libraries = [] for p in options.classpath: @@ -198,118 +350,48 @@ def main(args): if p not in libraries and p not in options.input_paths: libraries.append(p) - # TODO(agrieve): Remove proguard usages. - if options.r8_path: - temp_config_string = '' - with build_utils.TempDir() as tmp_dir: - tmp_mapping_path = os.path.join(tmp_dir, 'mapping.txt') - tmp_proguard_config_path = os.path.join(tmp_dir, 'proguard_config.txt') - # If there is no output (no classes are kept), this prevents this script - # from failing. - build_utils.Touch(tmp_mapping_path) - - f = cStringIO.StringIO() - proguard_util.WriteFlagsFile( - options.proguard_configs, f, exclude_generated=True) - merged_configs = f.getvalue() - # Fix up line endings (third_party configs can have windows endings) - merged_configs = merged_configs.replace('\r', '') - f.close() - print_stdout = '-whyareyoukeeping' in merged_configs - - def run_r8(cmd): - stderr_filter = None - env = os.environ.copy() - if options.disable_outlining: - stderr_filter = lambda l: re.sub(r'.*_JAVA_OPTIONS.*\n?', '', l) - env['_JAVA_OPTIONS'] = '-Dcom.android.tools.r8.disableOutlining=1' - build_utils.CheckOutput( - cmd, - env=env, - print_stdout=print_stdout, - stderr_filter=stderr_filter) - - if options.output_path.endswith('.dex'): - with build_utils.TempDir() as tmp_dex_dir: - cmd, temp_config_string = _CreateR8Command( - options, tmp_mapping_path, tmp_dex_dir, tmp_proguard_config_path, - libraries) - run_r8(cmd) - _MoveTempDexFile(tmp_dex_dir, options.output_path) - else: - cmd, temp_config_string = _CreateR8Command( - options, tmp_mapping_path, options.output_path, - tmp_proguard_config_path, libraries) - run_r8(cmd) - - # Copy output files to correct locations. - with build_utils.AtomicOutput(options.mapping_output) as mapping: - # Mapping files generated by R8 include comments that may break - # some of our tooling so remove those. - with open(tmp_mapping_path) as tmp: - mapping.writelines(l for l in tmp if not l.startswith('#')) - - for output in build_utils.ParseGnList(options.extra_mapping_output_paths): - shutil.copy(tmp_mapping_path, output) - - - with build_utils.AtomicOutput(options.output_config) as f: + _VerifyNoEmbeddedConfigs(options.input_paths + libraries) + + # ProGuard configs that are derived from flags. + dynamic_config_data = _CreateDynamicConfig(options) + + # ProGuard configs that are derived from flags. + merged_configs = _CombineConfigs( + options.proguard_configs, dynamic_config_data, exclude_generated=True) + print_stdout = '-whyareyoukeeping' in merged_configs or options.verbose + + # Writing the config output before we know ProGuard is going to succeed isn't + # great, since then a failure will result in one of the outputs being updated. + # We do it anyways though because the error message prints out the path to the + # config. Ninja will still know to re-run the command because of the other + # stale outputs. + if options.output_config: + with open(options.output_config, 'w') as f: f.write(merged_configs) - if temp_config_string: - f.write(_GENERATED_PROGUARD_HEADER) - f.write(temp_config_string) if options.expected_configs_file: _VerifyExpectedConfigs(options.expected_configs_file, options.output_config, options.verify_expected_configs) - other_inputs = [] - if options.apply_mapping: - other_inputs += options.apply_mapping - - build_utils.WriteDepfile( - options.depfile, - options.output_path, - inputs=options.proguard_configs + options.input_paths + libraries + - other_inputs, - add_pydeps=False) + if options.r8_path: + _OptimizeWithR8(options, options.proguard_configs, libraries, + dynamic_config_data, print_stdout) else: - proguard = proguard_util.ProguardCmdBuilder(options.proguard_path) - proguard.injars(options.input_paths) - proguard.configs(options.proguard_configs) - proguard.config_exclusions(options.proguard_config_exclusions) - proguard.outjar(options.output_path) - proguard.mapping_output(options.mapping_output) - proguard.libraryjars(libraries) - proguard.verbose(options.verbose) - proguard.min_api(options.min_api) - # Do not consider the temp file as an input since its name is random. - input_paths = proguard.GetInputs() - - with tempfile.NamedTemporaryFile() as f: - if options.apply_mapping: - input_paths.append(options.apply_mapping) - # Maintain only class name mappings in the .mapping file in order to - # work around what appears to be a ProGuard bug in -applymapping: - # method 'int close()' is not being kept as 'a', but remapped to 'c' - _RemoveMethodMappings(options.apply_mapping, f) - proguard.mapping(f.name) - with build_utils.TempDir() as d: - proguard.tmp_dir(d) - input_strings = proguard.build() - if f.name in input_strings: - input_strings[input_strings.index(f.name)] = '$M' - - build_utils.CallAndWriteDepfileIfStale( - proguard.CheckOutput, - options, - input_paths=input_paths, - input_strings=input_strings, - output_paths=proguard.GetOutputs(), - depfile_deps=proguard.GetDepfileDeps(), - add_pydeps=False) + _OptimizeWithProguard(options, options.proguard_configs, libraries, + dynamic_config_data, print_stdout) + + # After ProGuard / R8 has run: + for output in options.extra_mapping_output_paths: + shutil.copy(options.mapping_output, output) + + inputs = options.proguard_configs + options.input_paths + libraries + if options.apply_mapping: + inputs += options.apply_mapping + + build_utils.WriteDepfile( + options.depfile, options.output_path, inputs=inputs, add_pydeps=False) if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + main() diff --git a/chromium/build/android/gyp/proguard.pydeps b/chromium/build/android/gyp/proguard.pydeps index fd870a0e4b3..eb78ef367b3 100644 --- a/chromium/build/android/gyp/proguard.pydeps +++ b/chromium/build/android/gyp/proguard.pydeps @@ -6,4 +6,3 @@ util/__init__.py util/build_utils.py util/diff_utils.py util/md5_check.py -util/proguard_util.py diff --git a/chromium/build/android/gyp/util/build_utils.py b/chromium/build/android/gyp/util/build_utils.py index 4653d992ad8..ecb41a3fe85 100644 --- a/chromium/build/android/gyp/util/build_utils.py +++ b/chromium/build/android/gyp/util/build_utils.py @@ -577,7 +577,9 @@ def ExpandFileArgs(args): @FileArg(filename:key1:key2:...:keyn) The value of such a placeholder is calculated by reading 'filename' as json. - And then extracting the value at [key1][key2]...[keyn]. + And then extracting the value at [key1][key2]...[keyn]. If a key has a '[]' + suffix the (intermediate) value will be interpreted as a single item list and + the single item will be returned or used for further traversal. Note: This intentionally does not return the list of files that appear in such placeholders. An action that uses file-args *must* know the paths of those @@ -592,15 +594,25 @@ def ExpandFileArgs(args): if not match: continue + def get_key(key): + if key.endswith('[]'): + return key[:-2], True + return key, False + lookup_path = match.group(1).split(':') - file_path = lookup_path[0] + file_path, _ = get_key(lookup_path[0]) if not file_path in file_jsons: with open(file_path) as f: file_jsons[file_path] = json.load(f) - expansion = file_jsons[file_path] - for k in lookup_path[1:]: + expansion = file_jsons + for k in lookup_path: + k, flatten = get_key(k) expansion = expansion[k] + if flatten: + if not isinstance(expansion, list) or not len(expansion) == 1: + raise Exception('Expected single item list but got %s' % expansion) + expansion = expansion[0] # This should match ParseGnList. The output is either a GN-formatted list # or a literal (with no quotes). diff --git a/chromium/build/android/gyp/util/proguard_util.py b/chromium/build/android/gyp/util/proguard_util.py deleted file mode 100644 index 2945fdf1274..00000000000 --- a/chromium/build/android/gyp/util/proguard_util.py +++ /dev/null @@ -1,262 +0,0 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import os -import re -from util import build_utils - - -class ProguardOutputFilter(object): - """ProGuard outputs boring stuff to stdout (proguard version, jar path, etc) - as well as interesting stuff (notes, warnings, etc). If stdout is entirely - boring, this class suppresses the output. - """ - - IGNORE_RE = re.compile( - r'Pro.*version|Note:|Reading|Preparing|Printing|ProgramClass:|Searching|' - r'jar \[|\d+ class path entries checked') - - def __init__(self): - self._last_line_ignored = False - self._ignore_next_line = False - - def __call__(self, output): - ret = [] - for line in output.splitlines(True): - if self._ignore_next_line: - self._ignore_next_line = False - continue - - if '***BINARY RUN STATS***' in line: - self._last_line_ignored = True - self._ignore_next_line = True - elif not line.startswith(' '): - self._last_line_ignored = bool(self.IGNORE_RE.match(line)) - elif 'You should check if you need to specify' in line: - self._last_line_ignored = True - - if not self._last_line_ignored: - ret.append(line) - return ''.join(ret) - - -class ProguardCmdBuilder(object): - def __init__(self, proguard_jar): - assert os.path.exists(proguard_jar) - self._proguard_jar_path = proguard_jar - self._mapping = None - self._libraries = None - self._injars = None - self._configs = None - self._config_exclusions = None - self._outjar = None - self._mapping_output = None - self._verbose = False - self._min_api = None - self._tmp_dir = None - self._disabled_optimizations = [] - - def outjar(self, path): - assert self._outjar is None - self._outjar = path - - def mapping_output(self, path): - assert self._mapping_output is None - self._mapping_output = path - - def mapping(self, path): - assert self._mapping is None - assert os.path.exists(path), path - self._mapping = path - - def tmp_dir(self, path): - assert self._tmp_dir is None - self._tmp_dir = path - - def libraryjars(self, paths): - assert self._libraries is None - for p in paths: - assert os.path.exists(p), p - self._libraries = paths - - def injars(self, paths): - assert self._injars is None - for p in paths: - assert os.path.exists(p), p - self._injars = paths - - def configs(self, paths): - assert self._configs is None - self._configs = paths - for p in self._configs: - assert os.path.exists(p), p - - def config_exclusions(self, paths): - assert self._config_exclusions is None - self._config_exclusions = paths - - def verbose(self, verbose): - self._verbose = verbose - - def min_api(self, min_api): - assert self._min_api is None - self._min_api = min_api - - def disable_optimizations(self, optimizations): - self._disabled_optimizations += optimizations - - def build(self): - assert self._injars is not None - assert self._outjar is not None - assert self._configs is not None - - _combined_injars_path = os.path.join(self._tmp_dir, 'injars.jar') - _combined_libjars_path = os.path.join(self._tmp_dir, 'libjars.jar') - _combined_proguard_configs_path = os.path.join(self._tmp_dir, - 'includes.txt') - - build_utils.MergeZips(_combined_injars_path, self._injars) - build_utils.MergeZips(_combined_libjars_path, self._libraries) - _CombineConfigs(_combined_proguard_configs_path, self.GetConfigs()) - - if self._proguard_jar_path.endswith('.jar'): - cmd = [ - 'java', '-jar', self._proguard_jar_path, '-include', - _combined_proguard_configs_path - ] - else: - cmd = [self._proguard_jar_path, '@' + _combined_proguard_configs_path] - - if self._mapping: - cmd += ['-applymapping', self._mapping] - - if self._libraries: - cmd += ['-libraryjars', _combined_libjars_path] - - if self._min_api: - cmd += [ - '-assumevalues class android.os.Build$VERSION {' + - ' public static final int SDK_INT return ' + self._min_api + - '..9999; }' - ] - - for optimization in self._disabled_optimizations: - cmd += [ '-optimizations', '!' + optimization ] - - # The output jar must be specified after inputs. - cmd += [ - '-forceprocessing', - '-injars', - _combined_injars_path, - '-outjars', - self._outjar, - '-printseeds', - self._outjar + '.seeds', - '-printusage', - self._outjar + '.usage', - '-printmapping', - self._mapping_output, - ] - - if self._verbose: - cmd.append('-verbose') - - return cmd - - def GetDepfileDeps(self): - # The list of inputs that the GN target does not directly know about. - inputs = self._configs + self._injars - if self._libraries: - inputs += self._libraries - return inputs - - def GetConfigs(self): - ret = list(self._configs) - for path in self._config_exclusions: - ret.remove(path) - return ret - - def GetInputs(self): - inputs = self.GetDepfileDeps() - inputs += [self._proguard_jar_path] - if self._mapping: - inputs.append(self._mapping) - return inputs - - def GetOutputs(self): - return [ - self._outjar, - self._outjar + '.flags', - self._mapping_output, - self._outjar + '.seeds', - self._outjar + '.usage', - ] - - def _WriteFlagsFile(self, cmd, out): - # Quite useful for auditing proguard flags. - WriteFlagsFile(self._configs, out) - out.write('#' * 80 + '\n') - out.write('# Command-line\n') - out.write('#' * 80 + '\n') - out.write('# ' + ' '.join(cmd) + '\n') - - def CheckOutput(self): - cmd = self.build() - - # There are a couple scenarios (.mapping files and switching from no - # proguard -> proguard) where GN's copy() target is used on output - # paths. These create hardlinks, so we explicitly unlink here to avoid - # updating files with multiple links. - for path in self.GetOutputs(): - if os.path.exists(path): - os.unlink(path) - - with open(self._outjar + '.flags', 'w') as out: - self._WriteFlagsFile(cmd, out) - - # Warning: and Error: are sent to stderr, but messages and Note: are sent - # to stdout. - stdout_filter = None - stderr_filter = None - if not self._verbose: - stdout_filter = ProguardOutputFilter() - stderr_filter = ProguardOutputFilter() - build_utils.CheckOutput(cmd, print_stdout=True, - print_stderr=True, - stdout_filter=stdout_filter, - stderr_filter=stderr_filter) - - # Proguard will skip writing -printseeds / -printusage / -printmapping if - # the files would be empty, but ninja needs all outputs to exist. - open(self._outjar + '.seeds', 'a').close() - open(self._outjar + '.usage', 'a').close() - open(self._outjar + '.mapping', 'a').close() - - -def _CombineConfigs(output_config_path, input_configs): - # Combine all input_configs into one config file at output_config_path. - output_string = '' - for input_config in input_configs: - with open(input_config) as f_input_config: - output_string += f_input_config.read() - - with open(output_config_path, "w+") as f_output_config: - f_output_config.write(output_string) - - -def WriteFlagsFile(configs, out, exclude_generated=False): - for config in sorted(configs): - if exclude_generated and config.endswith('.resources.proguard.txt'): - continue - - out.write('#' * 80 + '\n') - out.write('# ' + config + '\n') - out.write('#' * 80 + '\n') - with open(config) as config_file: - contents = config_file.read().rstrip() - # Remove numbers from generated rule comments to make file more - # diff'able. - contents = re.sub(r' #generated:\d+', '', contents) - out.write(contents) - out.write('\n\n') diff --git a/chromium/build/android/gyp/util/zipalign.py b/chromium/build/android/gyp/util/zipalign.py index 0110df6d1ae..eb5aaed762a 100644 --- a/chromium/build/android/gyp/util/zipalign.py +++ b/chromium/build/android/gyp/util/zipalign.py @@ -69,7 +69,7 @@ def _SetAlignment(zip_obj, zip_info, alignment): """ cur_offset = zip_obj.fp.tell() header_size = _FIXED_ZIP_HEADER_LEN + len(zip_info.filename) - padding_needed = (header_size - cur_offset) % alignment + padding_needed = (cur_offset - header_size) % alignment # Extra field used to 4-byte align classes.dex. Alignment speeds up # execution when dex files are used via incremental install. diff --git a/chromium/build/android/gyp/write_build_config.py b/chromium/build/android/gyp/write_build_config.py index 40de1fe77e1..3d842a3fc5e 100755 --- a/chromium/build/android/gyp/write_build_config.py +++ b/chromium/build/android/gyp/write_build_config.py @@ -66,7 +66,6 @@ following required keys: * [android_app_bundle](#target_android_app_bundle) * [dist_jar](#target_dist_jar) * [dist_aar](#target_dist_aar) - * [resource_rewriter](#target_resource_rewriter) * [group](#target_group) See later sections for more details of some of these. @@ -503,15 +502,6 @@ This always has the following entries: -## <a name="target_resource_rewriter">Target type `resource_rewriter`</a>: - -The ResourceRewriter Java class is in charge of rewriting resource IDs at -runtime, for the benefit of the System WebView feature. This is a special -target type for it. - -Its `.build_config` only keeps a list of dependencies in its -`deps_info['deps_configs']` key. - ## <a name="dict_javac">The `deps_info['javac']` dictionary</a>: This dictionary appears in Java-related targets (e.g. `java_library`, @@ -561,7 +551,7 @@ from util import build_utils # Types that should never be used as a dependency of another build config. _ROOT_TYPES = ('android_apk', 'java_binary', 'java_annotation_processor', - 'junit_binary', 'resource_rewriter', 'android_app_bundle') + 'junit_binary', 'android_app_bundle') # Types that should not allow code deps to pass through. _RESOURCE_TYPES = ('android_assets', 'android_resources', 'system_java_library') @@ -1014,7 +1004,6 @@ def main(argv): 'java_binary': ['build_config'], 'java_library': ['build_config'] + jar_path_options, 'junit_binary': ['build_config'], - 'resource_rewriter': ['build_config'], 'system_java_library': ['build_config'], 'android_app_bundle': ['build_config', 'module_build_configs'], } @@ -1288,9 +1277,8 @@ def main(argv): config['javac']['resource_packages'] = [ c['package_name'] for c in all_resources_deps if 'package_name' in c] - if options.type in ( - 'android_resources', 'android_apk', 'junit_binary', 'resource_rewriter', - 'dist_aar', 'android_app_bundle_module'): + if options.type in ('android_resources', 'android_apk', 'junit_binary', + 'dist_aar', 'android_app_bundle_module'): dependency_zips = [ c['resources_zip'] for c in all_resources_deps if c['resources_zip'] @@ -1429,9 +1417,14 @@ def main(argv): GetDepConfig(c) for c in build_utils.ParseGnList(options.module_build_configs) ] - base_module_configs = [c for c in module_configs if c['is_base_module']] - assert len(base_module_configs) == 1, 'Must have exactly 1 base module!' - deps_info['base_module_config'] = base_module_configs[0]['path'] + jni_all_source = set() + for c in module_configs: + if c['is_base_module']: + assert 'base_module_config' not in deps_info, ( + 'Must have exactly 1 base module!') + deps_info['base_module_config'] = c['path'] + jni_all_source.update(c['jni']['all_source']) + deps_info['jni'] = {'all_source': sorted(jni_all_source)} # Map configs to classpath entries that should be included in their final dex. classpath_entries_by_owning_config = collections.defaultdict(list) @@ -1594,8 +1587,7 @@ def main(argv): # Exclude dex files from the test apk that exist within the apk under test. # TODO(agrieve): When proguard is enabled, this filtering logic happens - # within proguard_util.py. Move the logic for the proguard case into - # here as well. + # within proguard.py. Move the logic for the proguard case to here. tested_apk_library_deps = tested_apk_deps.All('java_library') tested_apk_deps_dex_files = [c['dex_path'] for c in tested_apk_library_deps] deps_dex_files = [ diff --git a/chromium/build/android/gyp/write_native_libraries_java.py b/chromium/build/android/gyp/write_native_libraries_java.py index c7ad066b1bf..244113a1118 100755 --- a/chromium/build/android/gyp/write_native_libraries_java.py +++ b/chromium/build/android/gyp/write_native_libraries_java.py @@ -30,10 +30,9 @@ def main(): parser.add_argument( '--enable-chromium-linker-tests', action='store_true', help='Run tests.') parser.add_argument( - '--native-libraries-list', help='File with list of native libraries.') + '--use-modern-linker', action='store_true', help='To use ModernLinker.') parser.add_argument( - '--exclude-native-libraries', - help='List of native libraries to exclude from the output.') + '--native-libraries-list', help='File with list of native libraries.') parser.add_argument( '--version-number', default='""', @@ -57,27 +56,16 @@ def main(): or not options.load_library_from_apk), ( 'Must set --enable-chromium-linker to load library from APK.') - lib_paths = [] - exclude_native_libraries = [] - if options.exclude_native_libraries: - exclude_native_libraries = options.exclude_native_libraries.split(',') + native_libraries_list = [] if options.native_libraries_list: with open(options.native_libraries_list) as f: - for line in f: - line = line.strip() - assert line.endswith('.so') - if os.path.basename(line) in exclude_native_libraries: - continue - lib_paths.append(line) - - def LibBasename(path): - filename = os.path.split(path)[1] - base = os.path.splitext(filename)[0] - return base[3:] # remove lib prefix - - # Convert to "base" library names: e.g. libfoo.so -> foo. - native_libraries_list = ( - '{%s}' % ','.join(['"%s"' % LibBasename(s) for s in lib_paths])) + for path in f: + path = path.strip() + filename = os.path.split(path)[1] + assert filename.startswith('lib') + assert filename.endswith('.so') + # Remove lib prefix and .so suffix. + native_libraries_list.append('"%s"' % filename[3:-3]) def bool_str(value): if value: @@ -91,7 +79,8 @@ def main(): 'USE_LINKER': bool_str(options.enable_chromium_linker), 'USE_LIBRARY_IN_ZIP_FILE': bool_str(options.load_library_from_apk), 'ENABLE_LINKER_TESTS': bool_str(options.enable_chromium_linker_tests), - 'LIBRARIES': native_libraries_list, + 'USE_MODERN_LINKER': bool_str(options.use_modern_linker), + 'LIBRARIES': ','.join(native_libraries_list), 'VERSION_NUMBER': options.version_number, 'CPU_FAMILY': options.cpu_family, } diff --git a/chromium/build/android/lint/suppressions.xml b/chromium/build/android/lint/suppressions.xml index e48e5616690..d8933a70cc7 100644 --- a/chromium/build/android/lint/suppressions.xml +++ b/chromium/build/android/lint/suppressions.xml @@ -206,6 +206,7 @@ Still reading? --> <issue id="MissingQuantity"> <ignore regexp="android_chrome_strings.xml"/> + <ignore regexp="android_chrome_tab_ui_strings.xml"/> </issue> <issue id="MissingRegistered" severity="ignore"/> <issue id="MissingSuperCall" severity="Error"> @@ -313,6 +314,19 @@ Still reading? <ignore regexp="chrome/android/chrome_strings_grd.resources.zip/values-vi/android_chrome_strings.xml"/> <ignore regexp="chrome/android/chrome_strings_grd.resources.zip/values-zh-rCN/android_chrome_strings.xml"/> <ignore regexp="chrome/android/chrome_strings_grd.resources.zip/values-zh-rTW/android_chrome_strings.xml"/> + + <!-- This is needed for suppressing warnings on upstream and downstream build bots --> + <ignore regexp="chrome/android/features/tab_ui/java_strings_grd.resources.zip/values-cs/android_chrome_tab_ui_strings.xml"/> + <ignore regexp="chrome/android/features/tab_ui/java_strings_grd.resources.zip/values-in/android_chrome_tab_ui_strings.xml"/> + <ignore regexp="chrome/android/features/tab_ui/java_strings_grd.resources.zip/values-ja/android_chrome_tab_ui_strings.xml"/> + <ignore regexp="chrome/android/features/tab_ui/java_strings_grd.resources.zip/values-ko/android_chrome_tab_ui_strings.xml"/> + <ignore regexp="chrome/android/features/tab_ui/java_strings_grd.resources.zip/values-lt/android_chrome_tab_ui_strings.xml"/> + <ignore regexp="chrome/android/features/tab_ui/java_strings_grd.resources.zip/values-ms/android_chrome_tab_ui_strings.xml"/> + <ignore regexp="chrome/android/features/tab_ui/java_strings_grd.resources.zip/values-sk/android_chrome_tab_ui_strings.xml"/> + <ignore regexp="chrome/android/features/tab_ui/java_strings_grd.resources.zip/values-th/android_chrome_tab_ui_strings.xml"/> + <ignore regexp="chrome/android/features/tab_ui/java_strings_grd.resources.zip/values-vi/android_chrome_tab_ui_strings.xml"/> + <ignore regexp="chrome/android/features/tab_ui/java_strings_grd.resources.zip/values-zh-rCN/android_chrome_tab_ui_strings.xml"/> + <ignore regexp="chrome/android/features/tab_ui/java_strings_grd.resources.zip/values-zh-rTW/android_chrome_tab_ui_strings.xml"/> </issue> <!-- Our generated enums are allowed to have the same values. --> <issue id="UniqueConstants" severity="ignore"/> @@ -344,6 +358,13 @@ Still reading? not use R.style.SplashTheme but new-style WebAPKs do. TODO(crbug.com/971254): Remove suppression once old-style WebAPKs are deprecated. --> <ignore regexp="The resource `R.style.SplashTheme` appears to be unused"/> + <!-- The WAM server currently has 2 codes paths for minting a WebAPK, and + it needs these "unused" resources. + TODO(crbug.com/1001115): Remove suppression once 2 code paths are merged --> + <ignore regexp="The resource `R.mipmap.ic_launcher_background` appears to be unused"/> + <ignore regexp="The resource `R.mipmap.ic_launcher_foreground` appears to be unused"/> + <ignore regexp="The resource `R.mipmap.maskable_splash_icon_xxhdpi` appears to be unused"/> + <ignore regexp="The resource `R.mipmap.maskable_splash_icon_xxxhdpi` appears to be unused"/> <!-- Endnote: Please specify number of suppressions when adding more --> </issue> <issue id="UseCompoundDrawables"> diff --git a/chromium/build/android/list_class_verification_failures.py b/chromium/build/android/list_class_verification_failures.py index 5ff5adcb569..cfcb2ac6896 100755 --- a/chromium/build/android/list_class_verification_failures.py +++ b/chromium/build/android/list_class_verification_failures.py @@ -124,7 +124,7 @@ def _AdbOatDumpForPackage(device, package_name, out_file): device.RunShellCommand(['oatdump', '--oat-file=' + odex_file, '--output=' + out_file], - shell=True, check_return=True) + timeout=120, shell=True, check_return=True) class JavaClass(object): diff --git a/chromium/build/android/pylib/base/environment_factory.py b/chromium/build/android/pylib/base/environment_factory.py index fdca803effa..4d3727444f4 100644 --- a/chromium/build/android/pylib/base/environment_factory.py +++ b/chromium/build/android/pylib/base/environment_factory.py @@ -4,12 +4,16 @@ from pylib import constants from pylib.local.device import local_device_environment +from pylib.local.emulator import local_emulator_environment from pylib.local.machine import local_machine_environment def CreateEnvironment(args, output_manager, error_func): if args.environment == 'local': if args.command not in constants.LOCAL_MACHINE_TESTS: + if args.avd_name: + return local_emulator_environment.LocalEmulatorEnvironment( + args, output_manager, error_func) return local_device_environment.LocalDeviceEnvironment( args, output_manager, error_func) else: diff --git a/chromium/build/android/pylib/constants/__init__.py b/chromium/build/android/pylib/constants/__init__.py index 3451a10d984..aff42e3e961 100644 --- a/chromium/build/android/pylib/constants/__init__.py +++ b/chromium/build/android/pylib/constants/__init__.py @@ -67,6 +67,18 @@ PACKAGE_INFO.update({ chrome.PackageInfo('com.android.webview', 'com.android.cts.webkit.WebViewStartupCtsActivity', 'webview-command-line', None), + 'android_google_webview_cts': + chrome.PackageInfo('com.google.android.webview', + 'com.android.cts.webkit.WebViewStartupCtsActivity', + 'webview-command-line', None), + 'android_system_webview_shell': + chrome.PackageInfo('org.chromium.webview_shell', + 'org.chromium.webview_shell.WebViewBrowserActivity', + 'webview-command-line', None), + 'android_webview_ui_test': + chrome.PackageInfo('org.chromium.webview_ui_test', + 'org.chromium.webview_ui_test.WebViewUiTestActivity', + 'webview-command-line', None), }) @@ -98,7 +110,7 @@ DEVICE_PERF_OUTPUT_DIR = ( SCREENSHOTS_DIR = os.path.join(DIR_SOURCE_ROOT, 'out_screenshots') ANDROID_SDK_VERSION = version_codes.OREO_MR1 -ANDROID_SDK_BUILD_TOOLS_VERSION = '27.0.3' +ANDROID_SDK_BUILD_TOOLS_VERSION = '29.0.2' ANDROID_SDK_ROOT = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'android_sdk', 'public') ANDROID_SDK_TOOLS = os.path.join(ANDROID_SDK_ROOT, diff --git a/chromium/build/android/pylib/gtest/gtest_test_instance.py b/chromium/build/android/pylib/gtest/gtest_test_instance.py index 6db67b7ccfb..2b2c5e7f7e9 100644 --- a/chromium/build/android/pylib/gtest/gtest_test_instance.py +++ b/chromium/build/android/pylib/gtest/gtest_test_instance.py @@ -29,13 +29,14 @@ BROWSER_TEST_SUITES = [ ] RUN_IN_SUB_THREAD_TEST_SUITES = [ - # Multiprocess tests should be run outside of the main thread. - 'base_unittests', # file_locking_unittest.cc uses a child process. - 'ipc_perftests', - 'ipc_tests', - 'mojo_perftests', - 'mojo_unittests', - 'net_unittests' + # Multiprocess tests should be run outside of the main thread. + 'base_unittests', # file_locking_unittest.cc uses a child process. + 'gwp_asan_unittests', + 'ipc_perftests', + 'ipc_tests', + 'mojo_perftests', + 'mojo_unittests', + 'net_unittests' ] diff --git a/chromium/build/android/pylib/instrumentation/instrumentation_test_instance.py b/chromium/build/android/pylib/instrumentation/instrumentation_test_instance.py index 343fb91df79..09114225cb1 100644 --- a/chromium/build/android/pylib/instrumentation/instrumentation_test_instance.py +++ b/chromium/build/android/pylib/instrumentation/instrumentation_test_instance.py @@ -40,13 +40,6 @@ _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS = [ _VALID_ANNOTATIONS = set(_DEFAULT_ANNOTATIONS + _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS) -# These test methods are inherited from android.test base test class and -# should be permitted for not having size annotation. For more, please check -# https://developer.android.com/reference/android/test/AndroidTestCase.html -# https://developer.android.com/reference/android/test/ServiceTestCase.html -_TEST_WITHOUT_SIZE_ANNOTATIONS = [ - 'testAndroidTestCaseSetupProperly', 'testServiceTestCaseSetUpProperly'] - _EXTRA_DRIVER_TEST_LIST = ( 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestList') _EXTRA_DRIVER_TEST_LIST_FILE = ( @@ -257,8 +250,7 @@ def FilterTests(tests, filter_str=None, annotations=None, continue # Enforce that all tests declare their size. - if (not any(a in _VALID_ANNOTATIONS for a in t['annotations']) - and t['method'] not in _TEST_WITHOUT_SIZE_ANNOTATIONS): + if not any(a in _VALID_ANNOTATIONS for a in t['annotations']): raise MissingSizeAnnotationError(GetTestName(t)) if (not annotation_filter(t['annotations']) diff --git a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py index e3a74934b11..18914e9aa13 100644 --- a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py +++ b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py @@ -79,6 +79,9 @@ EXTRA_TRACE_FILE = ('org.chromium.base.test.BaseJUnit4ClassRunner.TraceFile') _EXTRA_TEST_LIST = ( 'org.chromium.base.test.BaseChromiumAndroidJUnitRunner.TestList') +_EXTRA_PACKAGE_UNDER_TEST = ('org.chromium.chrome.test.pagecontroller.rules.' + 'ChromeUiApplicationTestRule.PackageUnderTest') + FEATURE_ANNOTATION = 'Feature' RENDER_TEST_FEATURE_ANNOTATION = 'RenderTest' @@ -395,9 +398,8 @@ class LocalDeviceInstrumentationTestRun( if self._test_instance.package_info: cmdline_file = self._test_instance.package_info.cmdline_file else: - logging.warning( - 'No PackageInfo found, falling back to using flag file %s', - cmdline_file) + raise Exception('No PackageInfo found but' + '--use-apk-under-test-flags-file is specified.') self._flag_changers[str(device)] = flag_changer.FlagChanger( device, cmdline_file) @@ -425,6 +427,11 @@ class LocalDeviceInstrumentationTestRun( def _RunTest(self, device, test): extras = {} + # Provide package name under test for apk_under_test. + if self._test_instance.apk_under_test: + package_name = self._test_instance.apk_under_test.GetPackageName() + extras[_EXTRA_PACKAGE_UNDER_TEST] = package_name + flags_to_add = [] test_timeout_scale = None if self._test_instance.coverage_directory: @@ -582,7 +589,7 @@ class LocalDeviceInstrumentationTestRun( def handle_coverage_data(): if self._test_instance.coverage_directory: try: - device.PullFile(coverage_directory, + device.PullFile(coverage_device_file, self._test_instance.coverage_directory) device.RunShellCommand( 'rm -f %s' % posixpath.join(coverage_directory, '*'), diff --git a/chromium/build/android/pylib/local/emulator/__init__.py b/chromium/build/android/pylib/local/emulator/__init__.py new file mode 100644 index 00000000000..4a12e35c925 --- /dev/null +++ b/chromium/build/android/pylib/local/emulator/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/chromium/build/android/pylib/local/emulator/local_emulator_environment.py b/chromium/build/android/pylib/local/emulator/local_emulator_environment.py new file mode 100644 index 00000000000..cd81cf9c3a7 --- /dev/null +++ b/chromium/build/android/pylib/local/emulator/local_emulator_environment.py @@ -0,0 +1,121 @@ +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import contextlib +import logging +import os +import socket +import stat + +from py_utils import tempfile_ext + +from devil.android.sdk import adb_wrapper +from devil.utils import cmd_helper +from devil.utils import timeout_retry + +from pylib import constants +from pylib.local.device import local_device_environment + + +class LocalEmulatorEnvironment(local_device_environment.LocalDeviceEnvironment): + + def __init__(self, args, output_manager, error_func): + super(LocalEmulatorEnvironment, self).__init__(args, output_manager, + error_func) + self._avd_name = args.avd_name + self._emulator_home = (args.emulator_home + or os.path.expanduser(os.path.join('~', '.android'))) + + root_ini = os.path.join(self._emulator_home, 'avd', + '%s.ini' % self._avd_name) + if not os.path.exists(root_ini): + error_func('Unable to find configuration for AVD %s at %s' % + (self._avd_name, root_ini)) + + self._emulator_path = os.path.join(constants.ANDROID_SDK_ROOT, 'emulator', + 'emulator') + if not os.path.exists(self._emulator_path): + error_func('%s does not exist.' % self._emulator_path) + + self._emulator_proc = None + self._emulator_serial = None + + #override + def SetUp(self): + # Emulator start-up looks for the adb daemon. Make sure it's running. + adb_wrapper.AdbWrapper.StartServer() + + # Emulator start-up tries to check for the SDK root by looking for + # platforms/ and platform-tools/. Ensure they exist. + # See http://bit.ly/2YAkyFE for context. + required_dirs = [ + os.path.join(constants.ANDROID_SDK_ROOT, 'platforms'), + os.path.join(constants.ANDROID_SDK_ROOT, 'platform-tools'), + ] + for d in required_dirs: + if not os.path.exists(d): + os.makedirs(d) + + # The emulator requires that some files are writable. + for dirname, _, filenames in os.walk(self._emulator_home): + for f in filenames: + path = os.path.join(dirname, f) + if (os.lstat(path).st_mode & + (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) == stat.S_IRUSR): + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + + self._emulator_proc, self._emulator_serial = self._StartInstance() + + logging.info('Emulator serial: %s', self._emulator_serial) + self._device_serials = [self._emulator_serial] + super(LocalEmulatorEnvironment, self).SetUp() + + def _StartInstance(self): + """Starts an AVD instance. + + Returns: + A (Popen, str) 2-tuple that includes the process and serial. + """ + # Start up the AVD. + with tempfile_ext.TemporaryFileName() as socket_path, (contextlib.closing( + socket.socket(socket.AF_UNIX))) as sock: + sock.bind(socket_path) + emulator_cmd = [ + self._emulator_path, + '-avd', + self._avd_name, + '-report-console', + 'unix:%s' % socket_path, + '-read-only', + '-no-window', + ] + emulator_env = {} + if self._emulator_home: + emulator_env['ANDROID_EMULATOR_HOME'] = self._emulator_home + sock.listen(1) + emulator_proc = cmd_helper.Popen(emulator_cmd, env=emulator_env) + + def listen_for_serial(s): + logging.info('Waiting for connection from emulator.') + with contextlib.closing(s.accept()[0]) as conn: + val = conn.recv(1024) + return 'emulator-%d' % int(val) + + try: + emulator_serial = timeout_retry.Run( + listen_for_serial, timeout=30, retries=0, args=[sock]) + except Exception: + emulator_proc.terminate() + raise + + return (emulator_proc, emulator_serial) + + #override + def TearDown(self): + try: + super(LocalEmulatorEnvironment, self).TearDown() + finally: + if self._emulator_proc: + self._emulator_proc.terminate() + self._emulator_proc.wait() diff --git a/chromium/build/android/pylib/utils/app_bundle_utils.py b/chromium/build/android/pylib/utils/app_bundle_utils.py index 2098f4f35d3..afae6456111 100644 --- a/chromium/build/android/pylib/utils/app_bundle_utils.py +++ b/chromium/build/android/pylib/utils/app_bundle_utils.py @@ -72,6 +72,10 @@ def GenerateBundleApks(bundle_path, or "system_compressed". """ device_spec = None + if minimal_sdk_version: + assert minimal or system_image_locales, ( + 'minimal_sdk_version is only used when minimal or system_image_locales ' + 'is specified') if minimal: # Measure with one language split installed. Use Hindi because it is # popular. resource_size.py looks for splits/base-hi.apk. diff --git a/chromium/build/android/resource_sizes.py b/chromium/build/android/resource_sizes.py index d843d1767ce..2067f51d0ca 100755 --- a/chromium/build/android/resource_sizes.py +++ b/chromium/build/android/resource_sizes.py @@ -613,6 +613,9 @@ def ResourceSizes(args): json.dump(chartjson, json_file, indent=2) + if json_file is not sys.stdout: + json_file.close() + # We would ideally generate a histogram set directly instead of generating # chartjson then converting. However, perf_tests_results_helper is in # //build, which doesn't seem to have any precedent for depending on diff --git a/chromium/build/android/stacktrace/crashpad_stackwalker.py b/chromium/build/android/stacktrace/crashpad_stackwalker.py index a538105be45..6be71cad897 100755 --- a/chromium/build/android/stacktrace/crashpad_stackwalker.py +++ b/chromium/build/android/stacktrace/crashpad_stackwalker.py @@ -46,6 +46,8 @@ def _CreateSymbolsDir(build_path, dynamic_library_names): build_path, '--binary', unstripped_library_path, + '--platform', + 'android', ] return_code = subprocess.call(cmd) if return_code != 0: diff --git a/chromium/build/android/test_runner.py b/chromium/build/android/test_runner.py index 043a53a5efe..443eb26221d 100755 --- a/chromium/build/android/test_runner.py +++ b/chromium/build/android/test_runner.py @@ -305,6 +305,20 @@ def AddDeviceOptions(parser): 'to the specified file.') +def AddEmulatorOptions(parser): + """Adds emulator-specific options to |parser|.""" + parser = parser.add_argument_group('emulator arguments') + + parser.add_argument( + '--avd-name', + help='Run and manage the lifetime of an AVD with the given name.') + parser.add_argument( + '--emulator-home', + type=os.path.realpath, + help='Emulator home directory ' + '(see ANDROID_EMULATOR_HOME: http://bit.ly/2K32oEy)') + + def AddGTestOptions(parser): """Adds gtest options to |parser|.""" @@ -877,6 +891,7 @@ def main(): help='googletest-based C++ tests') AddCommonOptions(subp) AddDeviceOptions(subp) + AddEmulatorOptions(subp) AddGTestOptions(subp) AddTracingOptions(subp) AddCommandLineOptions(subp) @@ -886,6 +901,7 @@ def main(): help='InstrumentationTestCase-based Java tests') AddCommonOptions(subp) AddDeviceOptions(subp) + AddEmulatorOptions(subp) AddInstrumentationTestOptions(subp) AddTracingOptions(subp) AddCommandLineOptions(subp) @@ -901,6 +917,7 @@ def main(): help='linker tests') AddCommonOptions(subp) AddDeviceOptions(subp) + AddEmulatorOptions(subp) AddLinkerTestOptions(subp) subp = command_parsers.add_parser( @@ -908,6 +925,7 @@ def main(): help="tests based on Android's monkey command") AddCommonOptions(subp) AddDeviceOptions(subp) + AddEmulatorOptions(subp) AddMonkeyTestOptions(subp) subp = command_parsers.add_parser( diff --git a/chromium/build/android/test_runner.pydeps b/chromium/build/android/test_runner.pydeps index 572cc3a1b5e..9b722c4027c 100644 --- a/chromium/build/android/test_runner.pydeps +++ b/chromium/build/android/test_runner.pydeps @@ -167,6 +167,8 @@ pylib/local/device/local_device_instrumentation_test_run.py pylib/local/device/local_device_linker_test_run.py pylib/local/device/local_device_monkey_test_run.py pylib/local/device/local_device_test_run.py +pylib/local/emulator/__init__.py +pylib/local/emulator/local_emulator_environment.py pylib/local/local_test_server_spawner.py pylib/local/machine/__init__.py pylib/local/machine/local_machine_environment.py |