diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2021-09-01 11:08:40 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2021-10-01 12:16:21 +0000 |
commit | 03c549e0392f92c02536d3f86d5e1d8dfa3435ac (patch) | |
tree | fe49d170a929b34ba82cd10db1a0bd8e3760fa4b /chromium/build/android | |
parent | 5d013f5804a0d91fcf6c626b2d6fb6eca5c845b0 (diff) |
BASELINE: Update Chromium to 91.0.4472.160
Change-Id: I0def1f08a2412aeed79a9ab95dd50eb5c3f65f31
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/build/android')
94 files changed, 1321 insertions, 708 deletions
diff --git a/chromium/build/android/BUILD.gn b/chromium/build/android/BUILD.gn index 826b0cf0928..1be9f479f20 100644 --- a/chromium/build/android/BUILD.gn +++ b/chromium/build/android/BUILD.gn @@ -22,6 +22,28 @@ if (enable_java_templates) { min_sdk_version = default_min_sdk_version } } + + generate_build_config_srcjar("build_config_gen") { + use_final_fields = false + } + + java_library("build_config_java") { + supports_android = true + srcjar_deps = [ ":build_config_gen" ] + jar_excluded_patterns = [ "*/build/BuildConfig.class" ] + } + + write_native_libraries_java("native_libraries_gen") { + use_final_fields = false + } + + android_library("native_libraries_java") { + srcjar_deps = [ ":native_libraries_gen" ] + + # New version of NativeLibraries.java (with the actual correct values) will + # be created when creating an apk. + jar_excluded_patterns = [ "*/NativeLibraries.class" ] + } } python_library("devil_chromium_py") { diff --git a/chromium/build/android/adb_command_line.py b/chromium/build/android/adb_command_line.py index 5d3e9ce11c0..c3ec8d49d03 100755 --- a/chromium/build/android/adb_command_line.py +++ b/chromium/build/android/adb_command_line.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/adb_install_apk.py b/chromium/build/android/adb_install_apk.py index 94d9a04bff7..6ec98e2f850 100755 --- a/chromium/build/android/adb_install_apk.py +++ b/chromium/build/android/adb_install_apk.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/chromium/build/android/adb_reverse_forwarder.py b/chromium/build/android/adb_reverse_forwarder.py index 3da9c98f77a..90d3139ae10 100755 --- a/chromium/build/android/adb_reverse_forwarder.py +++ b/chromium/build/android/adb_reverse_forwarder.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # # Copyright (c) 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/chromium/build/android/apk_operations.py b/chromium/build/android/apk_operations.py index 9d2cd865738..ff198098a63 100755 --- a/chromium/build/android/apk_operations.py +++ b/chromium/build/android/apk_operations.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -620,7 +620,7 @@ class _LogcatProcessor(object): # Logcat tags for messages that are generally relevant but are not from PIDs # associated with the apk. - _WHITELISTED_TAGS = { + _ALLOWLISTED_TAGS = { 'ActivityManager', # Shows activity lifecycle messages. 'ActivityTaskManager', # More activity lifecycle messages. 'AndroidRuntime', # Java crash dumps @@ -675,6 +675,15 @@ class _LogcatProcessor(object): # _start_pattern. There can be multiple "Start proc" messages from prior # runs of the app. self._found_initial_pid = self._primary_pid != None + # Retrieve any additional patterns that are relevant for the User. + self._user_defined_highlight = None + user_regex = os.environ['CHROMIUM_LOGCAT_HIGHLIGHT'] + if user_regex: + self._user_defined_highlight = re.compile(user_regex) + if not self._user_defined_highlight: + print(_Colorize( + 'Rejecting invalid regular expression: {}'.format(user_regex), + colorama.Fore.RED + colorama.Style.BRIGHT)) def _UpdateMyPids(self): # We intentionally do not clear self._my_pids to make sure that the @@ -741,10 +750,16 @@ class _LogcatProcessor(object): def _PrintParsedLine(self, parsed_line, dim=False): tid_style = colorama.Style.NORMAL + user_match = self._user_defined_highlight and ( + re.search(self._user_defined_highlight, parsed_line.tag) + or re.search(self._user_defined_highlight, parsed_line.message)) + # Make the main thread bright. if not dim and parsed_line.pid == parsed_line.tid: tid_style = colorama.Style.BRIGHT pid_style = self._GetPidStyle(parsed_line.pid, dim) + msg_style = pid_style if not user_match else (colorama.Fore.GREEN + + colorama.Style.BRIGHT) # We have to pad before adding color as that changes the width of the tag. pid_str = _Colorize('{:5}'.format(parsed_line.pid), pid_style) tid_str = _Colorize('{:5}'.format(parsed_line.tid), tid_style) @@ -756,7 +771,7 @@ class _LogcatProcessor(object): if self._deobfuscator: messages = self._deobfuscator.TransformLines(messages) for message in messages: - message = _Colorize(message, pid_style) + message = _Colorize(message, msg_style) sys.stdout.write('{} {} {} {} {} {}: {}\n'.format( parsed_line.date, parsed_line.invokation_time, pid_str, tid_str, priority, tag, message)) @@ -811,7 +826,7 @@ class _LogcatProcessor(object): return if owned_pid or self._verbose or (log.priority == 'F' or # Java crash dump - log.tag in self._WHITELISTED_TAGS): + log.tag in self._ALLOWLISTED_TAGS): if nonce_found: self._native_stack_symbolizer.AddLine(log, not owned_pid) else: diff --git a/chromium/build/android/apk_operations.pydeps b/chromium/build/android/apk_operations.pydeps index 3bbb036d334..60b128942e7 100644 --- a/chromium/build/android/apk_operations.pydeps +++ b/chromium/build/android/apk_operations.pydeps @@ -62,7 +62,7 @@ ../../third_party/catapult/devil/devil/utils/timeout_retry.py ../../third_party/catapult/devil/devil/utils/watchdog_timer.py ../../third_party/catapult/devil/devil/utils/zip_utils.py -../../third_party/catapult/third_party/zipfile/zipfile_2_7_13.py +../../third_party/catapult/third_party/six/six.py ../../third_party/jinja2/__init__.py ../../third_party/jinja2/_compat.py ../../third_party/jinja2/bccache.py diff --git a/chromium/build/android/apply_shared_preference_file.py b/chromium/build/android/apply_shared_preference_file.py index b2240819576..187bf18284d 100755 --- a/chromium/build/android/apply_shared_preference_file.py +++ b/chromium/build/android/apply_shared_preference_file.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # # Copyright 2018 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/chromium/build/android/bytecode/BUILD.gn b/chromium/build/android/bytecode/BUILD.gn index b56f341d90c..36b54329f65 100644 --- a/chromium/build/android/bytecode/BUILD.gn +++ b/chromium/build/android/bytecode/BUILD.gn @@ -19,16 +19,38 @@ java_binary("bytecode_processor") { enable_bytecode_checks = false } +# A bytecode rewriter that replaces all calls to +# `FragmentActivity Fragment.getActivity()` with +# `Activity Fragment.getActivity()`. java_binary("fragment_activity_replacer") { + main_class = "org.chromium.bytecode.FragmentActivityReplacer" + deps = [ ":fragment_activity_replacer_java" ] + wrapper_script_name = "helper/fragment_activity_replacer" +} + +# A bytecode rewriter that replaces all calls to +# `FragmentActivity Fragment.getActivity()` with +# `Activity Fragment.getActivity()` followed by a cast to FragmentActivity. +# Prefer :fragment_activity_replacer. This rewriter should only be used for +# libraries that rely on getActivity() returning a FragmentActivity *and* are +# not going to be used in an app that contains multiple copies of the AndroidX +# Fragment library (i.e. WebLayer). +java_binary("fragment_activity_replacer_single_androidx") { + main_class = "org.chromium.bytecode.FragmentActivityReplacer" + deps = [ ":fragment_activity_replacer_java" ] + wrapper_script_name = "helper/fragment_activity_replacer_single_androidx" + wrapper_script_args = [ "--single-androidx" ] +} + +java_library("fragment_activity_replacer_java") { + visibility = [ ":*" ] sources = [ "java/org/chromium/bytecode/ByteCodeRewriter.java", "java/org/chromium/bytecode/FragmentActivityReplacer.java", ] - main_class = "org.chromium.bytecode.FragmentActivityReplacer" deps = [ "//third_party/android_deps:org_ow2_asm_asm_commons_java", "//third_party/android_deps:org_ow2_asm_asm_java", "//third_party/android_deps:org_ow2_asm_asm_util_java", ] - wrapper_script_name = "helper/fragment_activity_replacer" } diff --git a/chromium/build/android/bytecode/java/org/chromium/bytecode/FragmentActivityReplacer.java b/chromium/build/android/bytecode/java/org/chromium/bytecode/FragmentActivityReplacer.java index 5180ff7698f..a40f39c4ce8 100644 --- a/chromium/build/android/bytecode/java/org/chromium/bytecode/FragmentActivityReplacer.java +++ b/chromium/build/android/bytecode/java/org/chromium/bytecode/FragmentActivityReplacer.java @@ -7,6 +7,7 @@ package org.chromium.bytecode; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; import org.objectweb.asm.commons.MethodRemapper; import org.objectweb.asm.commons.Remapper; @@ -26,16 +27,29 @@ public class FragmentActivityReplacer extends ByteCodeRewriter { private static final String OLD_METHOD_DESCRIPTOR = "()Landroidx/fragment/app/FragmentActivity;"; private static final String REQUIRE_ACTIVITY_METHOD_NAME = "requireActivity"; + private static final String SUPPORT_LIFECYCLE_FRAGMENT_IMPL_BINARY_NAME = + "com.google.android.gms.common.api.internal.SupportLifecycleFragmentImpl"; public static void main(String[] args) throws IOException { - // Invoke this script using //build/android/gyp/bytecode_processor.py - if (args.length != 2) { - System.err.println("Expected 2 arguments: [input.jar] [output.jar]"); + // Invoke this script using //build/android/gyp/bytecode_rewriter.py + if (!(args.length == 2 || args.length == 3 && args[0].equals("--single-androidx"))) { + System.err.println("Expected arguments: [--single-androidx] <input.jar> <output.jar>"); System.exit(1); } - FragmentActivityReplacer rewriter = new FragmentActivityReplacer(); - rewriter.rewrite(new File(args[0]), new File(args[1])); + if (args.length == 2) { + FragmentActivityReplacer rewriter = new FragmentActivityReplacer(false); + rewriter.rewrite(new File(args[0]), new File(args[1])); + } else { + FragmentActivityReplacer rewriter = new FragmentActivityReplacer(true); + rewriter.rewrite(new File(args[1]), new File(args[2])); + } + } + + private final boolean mSingleAndroidX; + + public FragmentActivityReplacer(boolean singleAndroidX) { + mSingleAndroidX = singleAndroidX; } @Override @@ -45,7 +59,7 @@ public class FragmentActivityReplacer extends ByteCodeRewriter { @Override protected ClassVisitor getClassVisitorForClass(String classPath, ClassVisitor delegate) { - ClassVisitor invocationVisitor = new InvocationReplacer(delegate); + ClassVisitor invocationVisitor = new InvocationReplacer(delegate, mSingleAndroidX); switch (classPath) { case "androidx/fragment/app/Fragment.class": return new FragmentClassVisitor(invocationVisitor); @@ -61,8 +75,11 @@ public class FragmentActivityReplacer extends ByteCodeRewriter { * the replaced method. */ private static class InvocationReplacer extends ClassVisitor { - private InvocationReplacer(ClassVisitor baseVisitor) { + private final boolean mSingleAndroidX; + + private InvocationReplacer(ClassVisitor baseVisitor, boolean singleAndroidX) { super(Opcodes.ASM7, baseVisitor); + mSingleAndroidX = singleAndroidX; } @Override @@ -73,17 +90,60 @@ public class FragmentActivityReplacer extends ByteCodeRewriter { @Override public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { - if ((opcode == Opcodes.INVOKEVIRTUAL || opcode == Opcodes.INVOKESPECIAL) + boolean isFragmentGetActivity = name.equals(GET_ACTIVITY_METHOD_NAME) && descriptor.equals(OLD_METHOD_DESCRIPTOR) - && (name.equals(GET_ACTIVITY_METHOD_NAME) - || name.equals(REQUIRE_ACTIVITY_METHOD_NAME) - || name.equals(GET_LIFECYCLE_ACTIVITY_METHOD_NAME))) { + && isFragmentSubclass(owner); + boolean isFragmentRequireActivity = name.equals(REQUIRE_ACTIVITY_METHOD_NAME) + && descriptor.equals(OLD_METHOD_DESCRIPTOR) + && isFragmentSubclass(owner); + boolean isSupportLifecycleFragmentImplGetLifecycleActivity = + name.equals(GET_LIFECYCLE_ACTIVITY_METHOD_NAME) + && descriptor.equals(OLD_METHOD_DESCRIPTOR) + && owner.equals(SUPPORT_LIFECYCLE_FRAGMENT_IMPL_BINARY_NAME); + if ((opcode == Opcodes.INVOKEVIRTUAL || opcode == Opcodes.INVOKESPECIAL) + && (isFragmentGetActivity || isFragmentRequireActivity + || isSupportLifecycleFragmentImplGetLifecycleActivity)) { super.visitMethodInsn( opcode, owner, name, NEW_METHOD_DESCRIPTOR, isInterface); + if (mSingleAndroidX) { + super.visitTypeInsn( + Opcodes.CHECKCAST, "androidx/fragment/app/FragmentActivity"); + } } else { super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); } } + + private boolean isFragmentSubclass(String internalType) { + // Look up classes with a ClassLoader that will resolve any R classes to Object. + // This is fine in this case as resource classes shouldn't be in the class + // hierarchy of any Fragments. + ClassLoader resourceStubbingClassLoader = new ClassLoader() { + @Override + protected Class<?> findClass(String name) throws ClassNotFoundException { + if (name.matches(".*\\.R(\\$.+)?")) { + return Object.class; + } + return super.findClass(name); + } + }; + + // This doesn't use Class#isAssignableFrom to avoid us needing to load + // AndroidX's Fragment class, which may not be on the classpath. + try { + String binaryName = Type.getObjectType(internalType).getClassName(); + Class<?> clazz = resourceStubbingClassLoader.loadClass(binaryName); + while (clazz != null) { + if (clazz.getName().equals("androidx.fragment.app.Fragment")) { + return true; + } + clazz = clazz.getSuperclass(); + } + return false; + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } }; } } @@ -127,6 +187,10 @@ public class FragmentActivityReplacer extends ByteCodeRewriter { "activityFromContext", "(Landroid/content/Context;)Landroid/app/Activity;", false); baseVisitor.visitInsn(Opcodes.ARETURN); + // Since we set COMPUTE_FRAMES, the arguments of visitMaxs are ignored, but calling + // it forces ClassWriter to actually recompute the correct stack/local values. + // Without this call ClassWriter keeps the original stack=0,locals=1 which is wrong. + baseVisitor.visitMaxs(0, 0); return null; } diff --git a/chromium/build/android/devil_chromium.pydeps b/chromium/build/android/devil_chromium.pydeps index 4b9c58dd9ea..41438059296 100644 --- a/chromium/build/android/devil_chromium.pydeps +++ b/chromium/build/android/devil_chromium.pydeps @@ -31,7 +31,7 @@ ../../third_party/catapult/devil/devil/utils/reraiser_thread.py ../../third_party/catapult/devil/devil/utils/timeout_retry.py ../../third_party/catapult/devil/devil/utils/watchdog_timer.py -../../third_party/catapult/third_party/zipfile/zipfile_2_7_13.py +../../third_party/catapult/third_party/six/six.py ../gn_helpers.py devil_chromium.py pylib/__init__.py diff --git a/chromium/build/android/docs/life_of_a_resource.md b/chromium/build/android/docs/life_of_a_resource.md index 4a30d270098..3aacd5e0c98 100644 --- a/chromium/build/android/docs/life_of_a_resource.md +++ b/chromium/build/android/docs/life_of_a_resource.md @@ -122,7 +122,7 @@ linked by the aapt2 link command which does the following: This step obfuscates / strips resources names from the resources.arsc so that they can be looked up only by their numeric ids (assigned in the compile resources step). Access to resources via `Resources.getIdentifier()` no longer -work unless resources are [whitelisted](#adding-resources-to-the-whitelist). +work unless resources are [allowlisted](#adding-resources-to-the-allowlist). ## App Bundles and Modules: @@ -159,9 +159,9 @@ For local builds, `R.txt` files are output in the `out/*/apks` directory. For official builds, Googlers can get archived `R.txt` files next to archived apks. -### Adding resources to the whitelist +### Adding resources to the allowlist -If a resource is accessed via `getIdentifier()` it needs to be whitelisted in an +If a resource is accessed via `getIdentifier()` it needs to be allowed by an aapt2 resources config file. The config file looks like this: ``` @@ -174,7 +174,7 @@ id/toolbar#no_obfuscate ``` The aapt2 config file is passed to the ninja target through the -`resources_config_paths` variable. To add a resource to the whitelist, check +`resources_config_paths` variable. To add a resource to the allowlist, check where the config is for your target and add a new line for your resource. If none exist, create a new config file and pass its path in your target. diff --git a/chromium/build/android/dump_apk_resource_strings.py b/chromium/build/android/dump_apk_resource_strings.py index b57db5032cd..8417e29f0f9 100755 --- a/chromium/build/android/dump_apk_resource_strings.py +++ b/chromium/build/android/dump_apk_resource_strings.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # encoding: utf-8 # Copyright 2019 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/chromium/build/android/emma_coverage_stats.py b/chromium/build/android/emma_coverage_stats.py index fe1775a8a24..f45f4d4d03b 100755 --- a/chromium/build/android/emma_coverage_stats.py +++ b/chromium/build/android/emma_coverage_stats.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/emma_coverage_stats_test.py b/chromium/build/android/emma_coverage_stats_test.py index d67f6be2180..d53292cbd73 100755 --- a/chromium/build/android/emma_coverage_stats_test.py +++ b/chromium/build/android/emma_coverage_stats_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/generate_jacoco_report.py b/chromium/build/android/generate_jacoco_report.py index f325ee7758a..d0a99871798 100755 --- a/chromium/build/android/generate_jacoco_report.py +++ b/chromium/build/android/generate_jacoco_report.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -100,6 +100,26 @@ def _GetFilesWithSuffix(root_dir, suffix): return files +def _GetExecFiles(root_dir, exclude_substr=None): + """ Gets all .exec files + + Args: + root_dir: Root directory in which to search for files. + exclude_substr: Substring which should be absent in filename. If None, all + files are selected. + + Returns: + A list of absolute paths to .exec files + + """ + all_exec_files = _GetFilesWithSuffix(root_dir, ".exec") + valid_exec_files = [] + for exec_file in all_exec_files: + if not exclude_substr or exclude_substr not in exec_file: + valid_exec_files.append(exec_file) + return valid_exec_files + + def _ParseArguments(parser): """Parses the command line arguments. @@ -129,6 +149,10 @@ def _ParseArguments(parser): required=True, help='Root of the directory in which to search for ' 'coverage data (.exec) files.') + parser.add_argument('--exec-filename-excludes', + required=False, + help='Excludes .exec files which contain a particular ' + 'substring in their name') parser.add_argument( '--sources-json-dir', help='Root of the directory in which to search for ' @@ -161,10 +185,8 @@ def _ParseArguments(parser): parser.error('--output-file needed for xml/csv reports.') if not args.device_or_host and args.sources_json_dir: parser.error('--device-or-host selection needed with --sources-json-dir') - if not (args.sources_json_dir or args.class_files): parser.error('At least either --sources-json-dir or --class-files needed.') - return args @@ -174,7 +196,7 @@ def main(): devil_chromium.Initialize() - coverage_files = _GetFilesWithSuffix(args.coverage_dir, '.exec') + coverage_files = _GetExecFiles(args.coverage_dir, args.exec_filename_excludes) if not coverage_files: parser.error('No coverage file found under %s' % args.coverage_dir) print('Found coverage files: %s' % str(coverage_files)) diff --git a/chromium/build/android/gtest_apk/BUILD.gn b/chromium/build/android/gtest_apk/BUILD.gn new file mode 100644 index 00000000000..2a72bc47ed2 --- /dev/null +++ b/chromium/build/android/gtest_apk/BUILD.gn @@ -0,0 +1,15 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/config/android/rules.gni") + +android_library("native_test_instrumentation_test_runner_java") { + testonly = true + sources = [ + "java/src/org/chromium/build/gtest_apk/NativeTestInstrumentationTestRunner.java", + "java/src/org/chromium/build/gtest_apk/NativeTestIntent.java", + "java/src/org/chromium/build/gtest_apk/TestStatusIntent.java", + "java/src/org/chromium/build/gtest_apk/TestStatusReceiver.java", + ] +} diff --git a/chromium/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestInstrumentationTestRunner.java b/chromium/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestInstrumentationTestRunner.java new file mode 100644 index 00000000000..652333bdd8e --- /dev/null +++ b/chromium/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestInstrumentationTestRunner.java @@ -0,0 +1,281 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.gtest_apk; + +import android.annotation.SuppressLint; +import android.app.Activity; +import android.app.ActivityManager; +import android.app.Instrumentation; +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.os.Bundle; +import android.os.Environment; +import android.os.Handler; +import android.os.Process; +import android.text.TextUtils; +import android.util.Log; +import android.util.SparseArray; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Queue; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * An Instrumentation that runs tests based on NativeTest. + */ +public class NativeTestInstrumentationTestRunner extends Instrumentation { + private static final String EXTRA_NATIVE_TEST_ACTIVITY = + "org.chromium.native_test.NativeTestInstrumentationTestRunner.NativeTestActivity"; + private static final String EXTRA_SHARD_NANO_TIMEOUT = + "org.chromium.native_test.NativeTestInstrumentationTestRunner.ShardNanoTimeout"; + private static final String EXTRA_SHARD_SIZE_LIMIT = + "org.chromium.native_test.NativeTestInstrumentationTestRunner.ShardSizeLimit"; + private static final String EXTRA_STDOUT_FILE = + "org.chromium.native_test.NativeTestInstrumentationTestRunner.StdoutFile"; + private static final String EXTRA_TEST_LIST_FILE = + "org.chromium.native_test.NativeTestInstrumentationTestRunner.TestList"; + private static final String EXTRA_TEST = + "org.chromium.native_test.NativeTestInstrumentationTestRunner.Test"; + + private static final String TAG = "NativeTest"; + + private static final long DEFAULT_SHARD_NANO_TIMEOUT = 60 * 1000000000L; + // Default to no size limit. + private static final int DEFAULT_SHARD_SIZE_LIMIT = 0; + + private Handler mHandler = new Handler(); + private Bundle mLogBundle = new Bundle(); + private SparseArray<ShardMonitor> mMonitors = new SparseArray<ShardMonitor>(); + private String mNativeTestActivity; + private TestStatusReceiver mReceiver; + private Queue<String> mShards = new ArrayDeque<String>(); + private long mShardNanoTimeout = DEFAULT_SHARD_NANO_TIMEOUT; + private int mShardSizeLimit = DEFAULT_SHARD_SIZE_LIMIT; + private File mStdoutFile; + private Bundle mTransparentArguments; + + @Override + public void onCreate(Bundle arguments) { + Context context = getContext(); + mTransparentArguments = new Bundle(arguments); + + mNativeTestActivity = arguments.getString(EXTRA_NATIVE_TEST_ACTIVITY); + if (mNativeTestActivity == null) { + Log.e(TAG, + "Unable to find org.chromium.native_test.NativeUnitTestActivity extra on " + + "NativeTestInstrumentationTestRunner launch intent."); + finish(Activity.RESULT_CANCELED, new Bundle()); + return; + } + mTransparentArguments.remove(EXTRA_NATIVE_TEST_ACTIVITY); + + String shardNanoTimeout = arguments.getString(EXTRA_SHARD_NANO_TIMEOUT); + if (shardNanoTimeout != null) mShardNanoTimeout = Long.parseLong(shardNanoTimeout); + mTransparentArguments.remove(EXTRA_SHARD_NANO_TIMEOUT); + + String shardSizeLimit = arguments.getString(EXTRA_SHARD_SIZE_LIMIT); + if (shardSizeLimit != null) mShardSizeLimit = Integer.parseInt(shardSizeLimit); + mTransparentArguments.remove(EXTRA_SHARD_SIZE_LIMIT); + + String stdoutFile = arguments.getString(EXTRA_STDOUT_FILE); + if (stdoutFile != null) { + mStdoutFile = new File(stdoutFile); + } else { + try { + mStdoutFile = File.createTempFile( + ".temp_stdout_", ".txt", Environment.getExternalStorageDirectory()); + Log.i(TAG, "stdout file created: " + mStdoutFile.getAbsolutePath()); + } catch (IOException e) { + Log.e(TAG, "Unable to create temporary stdout file.", e); + finish(Activity.RESULT_CANCELED, new Bundle()); + return; + } + } + + mTransparentArguments.remove(EXTRA_STDOUT_FILE); + + String singleTest = arguments.getString(EXTRA_TEST); + if (singleTest != null) { + mShards.add(singleTest); + } + + String testListFilePath = arguments.getString(EXTRA_TEST_LIST_FILE); + if (testListFilePath != null) { + File testListFile = new File(testListFilePath); + try { + BufferedReader testListFileReader = + new BufferedReader(new FileReader(testListFile)); + + String test; + ArrayList<String> workingShard = new ArrayList<String>(); + while ((test = testListFileReader.readLine()) != null) { + workingShard.add(test); + if (workingShard.size() == mShardSizeLimit) { + mShards.add(TextUtils.join(":", workingShard)); + workingShard = new ArrayList<String>(); + } + } + + if (!workingShard.isEmpty()) { + mShards.add(TextUtils.join(":", workingShard)); + } + + testListFileReader.close(); + } catch (IOException e) { + Log.e(TAG, "Error reading " + testListFile.getAbsolutePath(), e); + } + } + mTransparentArguments.remove(EXTRA_TEST_LIST_FILE); + + start(); + } + + @Override + @SuppressLint("DefaultLocale") + public void onStart() { + super.onStart(); + + mReceiver = new TestStatusReceiver(); + mReceiver.register(getContext()); + mReceiver.registerCallback(new TestStatusReceiver.TestRunCallback() { + @Override + public void testRunStarted(int pid) { + if (pid != Process.myPid()) { + ShardMonitor m = new ShardMonitor(pid, System.nanoTime() + mShardNanoTimeout); + mMonitors.put(pid, m); + mHandler.post(m); + } + } + + @Override + public void testRunFinished(int pid) { + ShardMonitor m = mMonitors.get(pid); + if (m != null) { + m.stopped(); + mMonitors.remove(pid); + } + mHandler.post(new ShardEnder(pid)); + } + + @Override + public void uncaughtException(int pid, String stackTrace) { + mLogBundle.putString(Instrumentation.REPORT_KEY_STREAMRESULT, + String.format("Uncaught exception in test process (pid: %d)%n%s%n", pid, + stackTrace)); + sendStatus(0, mLogBundle); + } + }); + + mHandler.post(new ShardStarter()); + } + + /** Monitors a test shard's execution. */ + private class ShardMonitor implements Runnable { + private static final int MONITOR_FREQUENCY_MS = 1000; + + private long mExpirationNanoTime; + private int mPid; + private AtomicBoolean mStopped; + + public ShardMonitor(int pid, long expirationNanoTime) { + mPid = pid; + mExpirationNanoTime = expirationNanoTime; + mStopped = new AtomicBoolean(false); + } + + public void stopped() { + mStopped.set(true); + } + + @Override + public void run() { + if (mStopped.get()) { + return; + } + + if (isAppProcessAlive(getContext(), mPid)) { + if (System.nanoTime() > mExpirationNanoTime) { + Log.e(TAG, String.format("Test process %d timed out.", mPid)); + mHandler.post(new ShardEnder(mPid)); + return; + } else { + mHandler.postDelayed(this, MONITOR_FREQUENCY_MS); + return; + } + } + + Log.e(TAG, String.format("Test process %d died unexpectedly.", mPid)); + mHandler.post(new ShardEnder(mPid)); + } + } + + private static boolean isAppProcessAlive(Context context, int pid) { + ActivityManager activityManager = + (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); + for (ActivityManager.RunningAppProcessInfo processInfo : + activityManager.getRunningAppProcesses()) { + if (processInfo.pid == pid) return true; + } + return false; + } + + protected Intent createShardMainIntent() { + Intent i = new Intent(Intent.ACTION_MAIN); + i.setComponent(new ComponentName(getContext().getPackageName(), mNativeTestActivity)); + i.setFlags(Intent.FLAG_ACTIVITY_NEW_TASK); + i.putExtras(mTransparentArguments); + if (mShards != null && !mShards.isEmpty()) { + String gtestFilter = mShards.remove(); + i.putExtra(NativeTestIntent.EXTRA_GTEST_FILTER, gtestFilter); + } + i.putExtra(NativeTestIntent.EXTRA_STDOUT_FILE, mStdoutFile.getAbsolutePath()); + return i; + } + + /** + * Starts the NativeTest Activity. + */ + private class ShardStarter implements Runnable { + @Override + public void run() { + getContext().startActivity(createShardMainIntent()); + } + } + + private class ShardEnder implements Runnable { + private static final int WAIT_FOR_DEATH_MILLIS = 10; + + private int mPid; + + public ShardEnder(int pid) { + mPid = pid; + } + + @Override + public void run() { + if (mPid != Process.myPid()) { + Process.killProcess(mPid); + try { + while (isAppProcessAlive(getContext(), mPid)) { + Thread.sleep(WAIT_FOR_DEATH_MILLIS); + } + } catch (InterruptedException e) { + Log.e(TAG, String.format("%d may still be alive.", mPid), e); + } + } + if (mShards != null && !mShards.isEmpty()) { + mHandler.post(new ShardStarter()); + } else { + finish(Activity.RESULT_OK, new Bundle()); + } + } + } +} diff --git a/chromium/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestIntent.java b/chromium/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestIntent.java new file mode 100644 index 00000000000..a875e9740e4 --- /dev/null +++ b/chromium/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestIntent.java @@ -0,0 +1,22 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.gtest_apk; + +/** + * Extras for intent sent by NativeTestInstrumentationTestRunner. + */ +public class NativeTestIntent { + public static final String EXTRA_COMMAND_LINE_FILE = + "org.chromium.native_test.NativeTest.CommandLineFile"; + public static final String EXTRA_COMMAND_LINE_FLAGS = + "org.chromium.native_test.NativeTest.CommandLineFlags"; + public static final String EXTRA_RUN_IN_SUB_THREAD = + "org.chromium.native_test.NativeTest.RunInSubThread"; + public static final String EXTRA_GTEST_FILTER = + "org.chromium.native_test.NativeTest.GtestFilter"; + public static final String EXTRA_STDOUT_FILE = "org.chromium.native_test.NativeTest.StdoutFile"; + public static final String EXTRA_COVERAGE_DEVICE_FILE = + "org.chromium.native_test.NativeTest.CoverageDeviceFile"; +} diff --git a/chromium/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusIntent.java b/chromium/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusIntent.java new file mode 100644 index 00000000000..520b7485b75 --- /dev/null +++ b/chromium/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusIntent.java @@ -0,0 +1,21 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.gtest_apk; + +/** + * Intent action and extras of broadcasts intercepted by TestStatusReceiver. + */ +public class TestStatusIntent { + public static final String ACTION_TEST_RUN_STARTED = + "org.chromium.test.reporter.TestStatusReporter.TEST_RUN_STARTED"; + public static final String ACTION_TEST_RUN_FINISHED = + "org.chromium.test.reporter.TestStatusReporter.TEST_RUN_FINISHED"; + public static final String ACTION_UNCAUGHT_EXCEPTION = + "org.chromium.test.reporter.TestStatusReporter.UNCAUGHT_EXCEPTION"; + public static final String DATA_TYPE_RESULT = "org.chromium.test.reporter/result"; + public static final String EXTRA_PID = "org.chromium.test.reporter.TestStatusReporter.PID"; + public static final String EXTRA_STACK_TRACE = + "org.chromium.test.reporter.TestStatusReporter.STACK_TRACE"; +} diff --git a/chromium/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusReceiver.java b/chromium/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusReceiver.java new file mode 100644 index 00000000000..e53900944ef --- /dev/null +++ b/chromium/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusReceiver.java @@ -0,0 +1,89 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.gtest_apk; + +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; +import android.content.IntentFilter; +import android.util.Log; + +import java.util.ArrayList; +import java.util.List; + +/** + Receives test status broadcasts sent from + {@link org.chromium.test.reporter.TestStatusReporter}. + */ +public class TestStatusReceiver extends BroadcastReceiver { + private static final String TAG = "test_reporter"; + + private final List<TestRunCallback> mTestRunCallbacks = new ArrayList<TestRunCallback>(); + + /** An IntentFilter that matches the intents that this class can receive. */ + private static final IntentFilter INTENT_FILTER; + static { + IntentFilter filter = new IntentFilter(); + filter.addAction(TestStatusIntent.ACTION_TEST_RUN_STARTED); + filter.addAction(TestStatusIntent.ACTION_TEST_RUN_FINISHED); + filter.addAction(TestStatusIntent.ACTION_UNCAUGHT_EXCEPTION); + try { + filter.addDataType(TestStatusIntent.DATA_TYPE_RESULT); + } catch (IntentFilter.MalformedMimeTypeException e) { + Log.wtf(TAG, "Invalid MIME type", e); + } + INTENT_FILTER = filter; + } + + /** A callback used when a test run has started or finished. */ + public interface TestRunCallback { + void testRunStarted(int pid); + void testRunFinished(int pid); + void uncaughtException(int pid, String stackTrace); + } + + /** Register a callback for when a test run has started or finished. */ + public void registerCallback(TestRunCallback c) { + mTestRunCallbacks.add(c); + } + + /** Register this receiver using the provided context. */ + public void register(Context c) { + c.registerReceiver(this, INTENT_FILTER); + } + + /** + * Receive a broadcast intent. + * + * @param context The Context in which the receiver is running. + * @param intent The intent received. + */ + @Override + public void onReceive(Context context, Intent intent) { + int pid = intent.getIntExtra(TestStatusIntent.EXTRA_PID, 0); + String stackTrace = intent.getStringExtra(TestStatusIntent.EXTRA_STACK_TRACE); + + switch (intent.getAction()) { + case TestStatusIntent.ACTION_TEST_RUN_STARTED: + for (TestRunCallback c : mTestRunCallbacks) { + c.testRunStarted(pid); + } + break; + case TestStatusIntent.ACTION_TEST_RUN_FINISHED: + for (TestRunCallback c : mTestRunCallbacks) { + c.testRunFinished(pid); + } + break; + case TestStatusIntent.ACTION_UNCAUGHT_EXCEPTION: + for (TestRunCallback c : mTestRunCallbacks) { + c.uncaughtException(pid, stackTrace); + } + break; + default: + Log.e(TAG, "Unrecognized intent received: " + intent.toString()); + break; + } + } +} diff --git a/chromium/build/android/gyp/assert_static_initializers.py b/chromium/build/android/gyp/assert_static_initializers.py index 3a2e7e3f5cf..856784d5b06 100755 --- a/chromium/build/android/gyp/assert_static_initializers.py +++ b/chromium/build/android/gyp/assert_static_initializers.py @@ -24,7 +24,7 @@ _DUMP_STATIC_INITIALIZERS_PATH = os.path.join(build_utils.DIR_SOURCE_ROOT, def _RunReadelf(so_path, options, tool_prefix=''): return subprocess.check_output([tool_prefix + 'readelf'] + options + - [so_path]) + [so_path]).decode('utf8') def _ParseLibBuildId(so_path, tool_prefix): diff --git a/chromium/build/android/gyp/bytecode_processor.py b/chromium/build/android/gyp/bytecode_processor.py index 89b05fc48ce..02465e080c7 100755 --- a/chromium/build/android/gyp/bytecode_processor.py +++ b/chromium/build/android/gyp/bytecode_processor.py @@ -6,11 +6,10 @@ """Wraps bin/helper/bytecode_processor and expands @FileArgs.""" import argparse -import os -import subprocess import sys from util import build_utils +from util import server_utils def _AddSwitch(parser, val): @@ -21,6 +20,7 @@ def _AddSwitch(parser, val): def main(argv): argv = build_utils.ExpandFileArgs(argv[1:]) parser = argparse.ArgumentParser() + parser.add_argument('--target-name', help='Fully qualified GN target name.') parser.add_argument('--script', required=True, help='Path to the java binary wrapper script.') parser.add_argument('--gn-target', required=True) @@ -38,6 +38,11 @@ def main(argv): _AddSwitch(parser, '--is-prebuilt') args = parser.parse_args(argv) + if server_utils.MaybeRunCommand(name=args.target_name, + argv=sys.argv, + stamp_file=args.stamp): + return + args.sdk_classpath_jars = build_utils.ParseGnList(args.sdk_classpath_jars) args.direct_classpath_jars = build_utils.ParseGnList( args.direct_classpath_jars) diff --git a/chromium/build/android/gyp/bytecode_processor.pydeps b/chromium/build/android/gyp/bytecode_processor.pydeps index a8a2370fa4f..6105d934da1 100644 --- a/chromium/build/android/gyp/bytecode_processor.pydeps +++ b/chromium/build/android/gyp/bytecode_processor.pydeps @@ -4,3 +4,4 @@ bytecode_processor.py util/__init__.py util/build_utils.py +util/server_utils.py diff --git a/chromium/build/android/gyp/check_flag_expectations.py b/chromium/build/android/gyp/check_flag_expectations.py new file mode 100644 index 00000000000..a6e47dbe18e --- /dev/null +++ b/chromium/build/android/gyp/check_flag_expectations.py @@ -0,0 +1,131 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse + +from util import build_utils +from util import diff_utils + +IGNORE_FLAG_PREFIXES = [ + # For cflags. + '-DANDROID_NDK_VERSION_ROLL', + '-DCR_LIBCXX_REVISION', + '-I', + '-g', + '-fcrash-diagnostics-dir=', + '-fprofile', + '--no-system-header-prefix', + '--system-header-prefix', + '-isystem', + '-iquote', + '-fmodule-map', + '-frandom-seed', + '-c ', + '-o ', + '-fmodule-name=', + '--sysroot=', + '-fcolor-diagnostics', + '-MF ', + '-MD', + + # For ldflags. + '-Wl,--thinlto-cache-dir', + '-Wl,--thinlto-cache-policy', + '-Wl,--thinlto-jobs', + '-Wl,--start-lib', + '-Wl,--end-lib', + '-Wl,-whole-archive', + '-Wl,-no-whole-archive', + '-l', + '-L', + '-Wl,-soname', + '-Wl,-version-script', + '-Wl,--version-script', + '-fdiagnostics-color', + '-Wl,--color-diagnostics', + '-B', + '-Wl,--dynamic-linker', + '-DCR_CLANG_REVISION=', +] + +FLAGS_WITH_PARAMS = ( + '-Xclang', + '-mllvm', + '-Xclang -fdebug-compilation-dir', + '-Xclang -add-plugin', +) + + +def KeepFlag(flag): + return not any(flag.startswith(prefix) for prefix in IGNORE_FLAG_PREFIXES) + + +def MergeFlags(flags): + flags = _MergeFlagsHelper(flags) + # For double params eg: -Xclang -fdebug-compilation-dir + flags = _MergeFlagsHelper(flags) + return flags + + +def _MergeFlagsHelper(flags): + merged_flags = [] + while flags: + current_flag = flags.pop(0) + if flags: + next_flag = flags[0] + else: + next_flag = None + merge_flags = False + + # Special case some flags that always come with params. + if current_flag in FLAGS_WITH_PARAMS: + merge_flags = True + # Assume flags without '-' are a param. + if next_flag and not next_flag.startswith('-'): + merge_flags = True + # Special case -plugin-arg prefix because it has the plugin name. + if current_flag.startswith('-Xclang -plugin-arg'): + merge_flags = True + if merge_flags: + merged_flag = '{} {}'.format(current_flag, next_flag) + merged_flags.append(merged_flag) + flags.pop(0) + else: + merged_flags.append(current_flag) + return merged_flags + + +def ParseFlags(flag_file_path): + flags = [] + with open(flag_file_path) as f: + for flag in f.read().splitlines(): + if KeepFlag(flag): + flags.append(flag) + return flags + + +def main(): + """Compare the flags with the checked in list.""" + parser = argparse.ArgumentParser() + diff_utils.AddCommandLineFlags(parser) + parser.add_argument('--current-flags', + help='Path to flags to check against expectations.') + options = parser.parse_args() + + flags = ParseFlags(options.current_flags) + flags = MergeFlags(flags) + + msg = """ +This expectation file is meant to inform the build team about changes to +flags used when building native libraries in chrome (most importantly any +that relate to security). This is to ensure the flags are replicated when +building native libraries outside of the repo. Please update the .expected +files and a WATCHLIST entry will alert the build team to your change.""" + diff_utils.CheckExpectations('\n'.join(sorted(flags)), + options, + custom_msg=msg) + + +if __name__ == '__main__': + main() diff --git a/chromium/build/android/gyp/check_flag_expectations.pydeps b/chromium/build/android/gyp/check_flag_expectations.pydeps new file mode 100644 index 00000000000..d8c394a04c1 --- /dev/null +++ b/chromium/build/android/gyp/check_flag_expectations.pydeps @@ -0,0 +1,7 @@ +# Generated by running: +# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/check_flag_expectations.pydeps build/android/gyp/check_flag_expectations.py +../../gn_helpers.py +check_flag_expectations.py +util/__init__.py +util/build_utils.py +util/diff_utils.py diff --git a/chromium/build/android/gyp/compile_java.py b/chromium/build/android/gyp/compile_java.py index 5d6616c7066..ea564c8ad14 100755 --- a/chromium/build/android/gyp/compile_java.py +++ b/chromium/build/android/gyp/compile_java.py @@ -358,7 +358,7 @@ class _InfoFileContext(object): entries = self._Collect() logging.info('Writing info file: %s', output_path) - with build_utils.AtomicOutput(output_path, mode='w') as f: + with build_utils.AtomicOutput(output_path, mode='wb') as f: jar_info_utils.WriteJarInfoFile(f, entries, self._srcjar_files) logging.info('Completed info file: %s', output_path) @@ -528,6 +528,9 @@ def _ParseOptions(argv): build_utils.AddDepfileOption(parser) parser.add_option('--target-name', help='Fully qualified GN target name.') + parser.add_option('--skip-build-server', + action='store_true', + help='Avoid using the build server.') parser.add_option( '--java-srcjars', action='append', @@ -638,8 +641,10 @@ def main(argv): options, java_files = _ParseOptions(argv) # Only use the build server for errorprone runs. - if options.enable_errorprone and server_utils.MaybeRunCommand( - name=options.target_name, argv=sys.argv, stamp_file=options.jar_path): + if (options.enable_errorprone and not options.skip_build_server + and server_utils.MaybeRunCommand(name=options.target_name, + argv=sys.argv, + stamp_file=options.jar_path)): return colorama.init() diff --git a/chromium/build/android/gyp/copy_ex.py b/chromium/build/android/gyp/copy_ex.py index f93597f973f..3fb1a362d2a 100755 --- a/chromium/build/android/gyp/copy_ex.py +++ b/chromium/build/android/gyp/copy_ex.py @@ -74,7 +74,7 @@ def DoRenaming(options, deps): print('Renaming source and destination files not match.') sys.exit(-1) - for src, dest in itertools.izip(src_files, dest_files): + for src, dest in zip(src_files, dest_files): if os.path.isdir(src): print('renaming diretory is not supported.') sys.exit(-1) diff --git a/chromium/build/android/gyp/create_bundle_wrapper_script.py b/chromium/build/android/gyp/create_bundle_wrapper_script.py index b28a6abbcd2..f4f4cc68459 100755 --- a/chromium/build/android/gyp/create_bundle_wrapper_script.py +++ b/chromium/build/android/gyp/create_bundle_wrapper_script.py @@ -114,7 +114,7 @@ def main(args): repr(args.default_modules), } script.write(SCRIPT_TEMPLATE.substitute(script_dict)) - os.chmod(args.script_output_path, 0750) + os.chmod(args.script_output_path, 0o750) return 0 diff --git a/chromium/build/android/gyp/create_size_info_files.py b/chromium/build/android/gyp/create_size_info_files.py index b446b7f5dd4..f33d608420d 100755 --- a/chromium/build/android/gyp/create_size_info_files.py +++ b/chromium/build/android/gyp/create_size_info_files.py @@ -36,7 +36,8 @@ def _TransformAarPaths(path): def _MergeResInfoFiles(res_info_path, info_paths): # Concatenate them all. # only_if_changed=False since no build rules depend on this as an input. - with build_utils.AtomicOutput(res_info_path, only_if_changed=False) as dst: + with build_utils.AtomicOutput(res_info_path, only_if_changed=False, + mode='w+') as dst: for p in info_paths: with open(p) as src: dst.writelines(_TransformAarPaths(l) for l in src) @@ -52,7 +53,8 @@ def _MergePakInfoFiles(merged_path, pak_infos): with open(pak_info_path, 'r') as src_info_file: info_lines.update(_TransformAarPaths(x) for x in src_info_file) # only_if_changed=False since no build rules depend on this as an input. - with build_utils.AtomicOutput(merged_path, only_if_changed=False) as f: + with build_utils.AtomicOutput(merged_path, only_if_changed=False, + mode='w+') as f: f.writelines(sorted(info_lines)) diff --git a/chromium/build/android/gyp/dex.py b/chromium/build/android/gyp/dex.py index 4618369ffd7..996bf80588d 100755 --- a/chromium/build/android/gyp/dex.py +++ b/chromium/build/android/gyp/dex.py @@ -30,10 +30,14 @@ _IGNORE_WARNINGS = ( r'Type `dalvik.system.VMStack` was not found', # Caused by jacoco code coverage: r'Type `java.lang.management.ManagementFactory` was not found', - # Filter out warnings caused by our fake main dex list used to enable - # multidex on library targets. - # Warning: Application does not contain `Foo` as referenced in main-dex-list - r'does not contain `Foo`', + # TODO(wnwen): Remove this after R8 version 3.0.26-dev: + r'Missing class sun.misc.Unsafe', + # Caused when the test apk and the apk under test do not having native libs. + r'Missing class org.chromium.build.NativeLibraries', + # Caused by internal annotation: https://crbug.com/1180222 + r'Missing class com.google.errorprone.annotations.RestrictedInheritance', + # Caused by internal protobuf package: https://crbug.com/1183971 + r'referenced from: com.google.protobuf.GeneratedMessageLite$GeneratedExtension', # pylint: disable=line-too-long ) @@ -71,6 +75,11 @@ def _ParseArgs(args): action='store_true', help='Allow numerous dex files within output.') parser.add_argument('--r8-jar-path', required=True, help='Path to R8 jar.') + parser.add_argument('--skip-custom-d8', + action='store_true', + help='When rebuilding the CustomD8 jar, this may be ' + 'necessary to avoid incompatibility with the new r8 ' + 'jar.') parser.add_argument('--custom-d8-jar-path', required=True, help='Path to our customized d8 jar.') @@ -104,6 +113,10 @@ def _ParseArgs(args): parser.add_argument('--warnings-as-errors', action='store_true', help='Treat all warnings as errors.') + parser.add_argument('--dump-inputs', + action='store_true', + help='Use when filing D8 bugs to capture inputs.' + ' Stores inputs to d8inputs.zip') group = parser.add_argument_group('Dexlayout') group.add_argument( @@ -364,17 +377,9 @@ def _CreateFinalDex(d8_inputs, output, tmp_dir, dex_cmd, options=None): needs_dexing = not all(f.endswith('.dex') for f in d8_inputs) needs_dexmerge = output.endswith('.dex') or not (options and options.library) if needs_dexing or needs_dexmerge: - if options: - if options.main_dex_rules_path: - for main_dex_rule in options.main_dex_rules_path: - dex_cmd = dex_cmd + ['--main-dex-rules', main_dex_rule] - elif options.library and int(options.min_api or 1) < 21: - # When dexing D8 requires a main dex list pre-21. For library targets, - # it doesn't matter what's in the main dex, so just use a dummy one. - tmp_main_dex_list_path = os.path.join(tmp_dir, 'main_list.txt') - with open(tmp_main_dex_list_path, 'w') as f: - f.write('Foo.class\n') - dex_cmd = dex_cmd + ['--main-dex-list', tmp_main_dex_list_path] + if options and options.main_dex_rules_path: + for main_dex_rule in options.main_dex_rules_path: + dex_cmd = dex_cmd + ['--main-dex-rules', main_dex_rule] tmp_dex_dir = os.path.join(tmp_dir, 'tmp_dex_dir') os.mkdir(tmp_dex_dir) @@ -502,7 +507,7 @@ def _CreateIntermediateDexFiles(changes, options, tmp_dir, dex_cmd): if class_files: # Dex necessary classes into intermediate dex files. dex_cmd = dex_cmd + ['--intermediate', '--file-per-class-file'] - if options.desugar_dependencies: + if options.desugar_dependencies and not options.skip_custom_d8: dex_cmd += ['--file-tmp-prefix', tmp_extract_dir] _RunD8(dex_cmd, class_files, options.incremental_dir, options.warnings_as_errors, @@ -563,11 +568,24 @@ def main(args): final_dex_inputs = list(options.class_inputs) final_dex_inputs += options.dex_inputs - dex_cmd = build_utils.JavaCmd(options.warnings_as_errors) + [ - '-cp', - '{}:{}'.format(options.r8_jar_path, options.custom_d8_jar_path), - 'org.chromium.build.CustomD8', - ] + dex_cmd = build_utils.JavaCmd(options.warnings_as_errors) + + if options.dump_inputs: + dex_cmd += ['-Dcom.android.tools.r8.dumpinputtofile=d8inputs.zip'] + + if not options.skip_custom_d8: + dex_cmd += [ + '-cp', + '{}:{}'.format(options.r8_jar_path, options.custom_d8_jar_path), + 'org.chromium.build.CustomD8', + ] + else: + dex_cmd += [ + '-cp', + options.r8_jar_path, + 'com.android.tools.r8.D8', + ] + if options.release: dex_cmd += ['--release'] if options.min_api: @@ -577,7 +595,7 @@ def main(args): dex_cmd += ['--no-desugaring'] elif options.classpath: # The classpath is used by D8 to for interface desugaring. - if options.desugar_dependencies: + if options.desugar_dependencies and not options.skip_custom_d8: dex_cmd += ['--desugar-dependencies', options.desugar_dependencies] if track_subpaths_allowlist: track_subpaths_allowlist += options.classpath diff --git a/chromium/build/android/gyp/extract_unwind_tables.py b/chromium/build/android/gyp/extract_unwind_tables.py index b20f740764a..edfd1ef5f3e 100755 --- a/chromium/build/android/gyp/extract_unwind_tables.py +++ b/chromium/build/android/gyp/extract_unwind_tables.py @@ -135,6 +135,7 @@ def _GetAllCfiRows(symbol_file): cfi_data = {} current_func = [] for line in symbol_file: + line = line.decode('utf8') if 'STACK CFI' not in line: continue @@ -197,7 +198,7 @@ def _WriteCfiData(cfi_data, out_file): # Store mapping between the functions to the index. func_addr_to_index = {} previous_func_end = 0 - for addr, function in sorted(cfi_data.iteritems()): + for addr, function in sorted(cfi_data.items()): # Add an empty function entry when functions CFIs are missing between 2 # functions. if previous_func_end != 0 and addr - previous_func_end > 4: @@ -211,7 +212,7 @@ def _WriteCfiData(cfi_data, out_file): # rows have CFI data. Create function data array as given in the format. for row in function[1:]: addr_offset = row[_ADDR_ENTRY] - addr - cfa_offset = (row[_CFA_REG]) | (row[_RA_REG] / 4) + cfa_offset = (row[_CFA_REG]) | (row[_RA_REG] // 4) func_data_arr.append(addr_offset) func_data_arr.append(cfa_offset) @@ -221,7 +222,7 @@ def _WriteCfiData(cfi_data, out_file): for data in func_data_arr: func_data = (func_data << 16) | data - row_count = len(func_data_arr) / 2 + row_count = len(func_data_arr) // 2 if func_data not in data_to_index: # When data is not found, create a new index = len(unw_data), and write # the data to |unw_data|. @@ -243,7 +244,7 @@ def _WriteCfiData(cfi_data, out_file): _Write4Bytes(out_file, len(func_addr_to_index)) # Write the UNW_INDEX table. First list of addresses and then indices. - sorted_unw_index = sorted(func_addr_to_index.iteritems()) + sorted_unw_index = sorted(func_addr_to_index.items()) for addr, index in sorted_unw_index: _Write4Bytes(out_file, addr) for addr, index in sorted_unw_index: diff --git a/chromium/build/android/gyp/lint.py b/chromium/build/android/gyp/lint.py index c193927bfeb..b30c6720054 100755 --- a/chromium/build/android/gyp/lint.py +++ b/chromium/build/android/gyp/lint.py @@ -358,6 +358,9 @@ def _ParseArgs(argv): parser = argparse.ArgumentParser() build_utils.AddDepfileOption(parser) parser.add_argument('--target-name', help='Fully qualified GN target name.') + parser.add_argument('--skip-build-server', + action='store_true', + help='Avoid using the build server.') parser.add_argument('--lint-binary-path', required=True, help='Path to lint executable.') @@ -437,8 +440,9 @@ def main(): # invocations. # Avoid parallelizing cache creation since lint runs without the cache defeat # the purpose of creating the cache in the first place. - if not args.create_cache and server_utils.MaybeRunCommand( - name=args.target_name, argv=sys.argv, stamp_file=args.stamp): + if (not args.create_cache and not args.skip_build_server + and server_utils.MaybeRunCommand( + name=args.target_name, argv=sys.argv, stamp_file=args.stamp)): return sources = [] diff --git a/chromium/build/android/gyp/native_libraries_template.py b/chromium/build/android/gyp/native_libraries_template.py index 2637ee3ead9..cf336ecf492 100644 --- a/chromium/build/android/gyp/native_libraries_template.py +++ b/chromium/build/android/gyp/native_libraries_template.py @@ -7,7 +7,7 @@ NATIVE_LIBRARIES_TEMPLATE = """\ // build/android/gyp/write_native_libraries_java.py // Please do not change its content. -package org.chromium.base.library_loader; +package org.chromium.build; public class NativeLibraries {{ public static final int CPU_FAMILY_UNKNOWN = 0; diff --git a/chromium/build/android/gyp/prepare_resources.py b/chromium/build/android/gyp/prepare_resources.py index 6e3b1aa8d0c..4f627ff73a0 100755 --- a/chromium/build/android/gyp/prepare_resources.py +++ b/chromium/build/android/gyp/prepare_resources.py @@ -39,12 +39,6 @@ def _ParseArgs(args): 'in the generated R.txt when generating R.java.') parser.add_argument( - '--shared-resources', - action='store_true', - help='Make resources shareable by generating an onResourcesLoaded() ' - 'method in the R.java source file.') - - parser.add_argument( '--resource-zip-out', help='Path to a zip archive containing all resources from ' '--resource-dirs, merged into a single directory tree.') @@ -188,7 +182,6 @@ def main(args): # Resource filenames matter to the output, so add them to strings as well. # This matters if a file is renamed but not changed (http://crbug.com/597126). input_strings = sorted(resource_names) + [ - options.shared_resources, options.strip_drawables, ] diff --git a/chromium/build/android/gyp/proguard.py b/chromium/build/android/gyp/proguard.py index 9a257fd516d..c22a3fd912a 100755 --- a/chromium/build/android/gyp/proguard.py +++ b/chromium/build/android/gyp/proguard.py @@ -31,6 +31,7 @@ _API_LEVEL_VERSION_CODE = [ (28, 'P'), (29, 'Q'), (30, 'R'), + (31, 'S'), ] @@ -302,7 +303,8 @@ def _OptimizeWithR8(options, tmp_output) base_context = split_contexts_by_name['base'] - cmd = build_utils.JavaCmd(options.warnings_as_errors) + [ + # R8 OOMs with the default xmx=1G. + cmd = build_utils.JavaCmd(options.warnings_as_errors, xmx='2G') + [ '-Dcom.android.tools.r8.allowTestProguardOptions=1', '-Dcom.android.tools.r8.verticalClassMerging=1', ] @@ -488,7 +490,7 @@ def _CheckForMissingSymbols(r8_path, dex_files, classpath, warnings_as_errors): # trichrome_webview_google_bundle contains this missing reference. # TODO(crbug.com/1142530): Fix this missing reference properly. - 'org/chromium/base/library_loader/NativeLibraries', + 'org/chromium/build/NativeLibraries', # TODO(agrieve): Exclude these only when use_jacoco_coverage=true. 'Ljava/lang/instrument/ClassFileTransformer', @@ -553,15 +555,19 @@ def _CombineConfigs(configs, dynamic_config_data, exclude_generated=False): if exclude_generated and config.endswith('.resources.proguard.txt'): continue - ret.append('# File: ' + config) with open(config) as config_file: contents = config_file.read().rstrip() + if not contents.strip(): + # Ignore empty files. + continue + # Fix up line endings (third_party configs can have windows endings). contents = contents.replace('\r', '') # Remove numbers from generated rule comments to make file more # diff'able. contents = re.sub(r' #generated:\d+', '', contents) + ret.append('# File: ' + config) ret.append(contents) ret.append('') @@ -573,7 +579,11 @@ def _CombineConfigs(configs, dynamic_config_data, exclude_generated=False): def _CreateDynamicConfig(options): - ret = [] + # Our scripts already fail on output. Adding -ignorewarnings makes R8 output + # warnings rather than throw exceptions so we can selectively ignore them via + # dex.py's ignore list. Context: https://crbug.com/1180222 + ret = ["-ignorewarnings"] + if options.sourcefile: ret.append("-renamesourcefileattribute '%s' # OMIT FROM EXPECTATIONS" % options.sourcefile) diff --git a/chromium/build/android/gyp/util/build_utils_test.py b/chromium/build/android/gyp/util/build_utils_test.py index d462f0c6769..75a3ba32ac2 100755 --- a/chromium/build/android/gyp/util/build_utils_test.py +++ b/chromium/build/android/gyp/util/build_utils_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2018 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/gyp/util/diff_utils.py b/chromium/build/android/gyp/util/diff_utils.py index f746df3be6a..a109100c7d2 100755 --- a/chromium/build/android/gyp/util/diff_utils.py +++ b/chromium/build/android/gyp/util/diff_utils.py @@ -88,9 +88,10 @@ def AddCommandLineFlags(parser): help='Verify the expectation and exit.') -def CheckExpectations(actual_data, options): - with build_utils.AtomicOutput(options.actual_file) as f: - f.write(actual_data) +def CheckExpectations(actual_data, options, custom_msg=''): + if options.actual_file: + with build_utils.AtomicOutput(options.actual_file) as f: + f.write(actual_data.encode('utf8')) if options.expected_file_base: actual_data = _GenerateDiffWithOnlyAdditons(options.expected_file_base, actual_data) @@ -106,13 +107,15 @@ https://chromium.googlesource.com/chromium/src/+/HEAD/chrome/android/expectation LogDog tip: Use "Raw log" or "Switch to lite mode" before copying: https://bugs.chromium.org/p/chromium/issues/detail?id=984616 +{} + To update expectations, run: ########### START ########### patch -p1 <<'END_DIFF' {} END_DIFF ############ END ############ -""".format(diff_text) +""".format(custom_msg, diff_text) sys.stderr.write(fail_msg) diff --git a/chromium/build/android/gyp/util/jar_info_utils.py b/chromium/build/android/gyp/util/jar_info_utils.py index 16a3af6f46b..975945510e3 100644 --- a/chromium/build/android/gyp/util/jar_info_utils.py +++ b/chromium/build/android/gyp/util/jar_info_utils.py @@ -55,4 +55,5 @@ def WriteJarInfoFile(output_obj, info_data, source_file_map=None): path = source_file_map[path] assert not path.startswith('/tmp'), ( 'Java file path should not be in temp dir: {}'.format(path)) - output_obj.write('{},{}\n'.format(fully_qualified_name, path)) + output_obj.write(('{},{}\n'.format(fully_qualified_name, + path)).encode('utf8')) diff --git a/chromium/build/android/gyp/util/manifest_utils.py b/chromium/build/android/gyp/util/manifest_utils.py index e594958c2a9..1a354b9ab73 100644 --- a/chromium/build/android/gyp/util/manifest_utils.py +++ b/chromium/build/android/gyp/util/manifest_utils.py @@ -129,7 +129,7 @@ def _SortAndStripElementTree(root): # for the node or any of its descendants. Remove them so as to prevent a # change to a child that adds/removes a namespace usage from changing sort # order. - return re.sub(r' xmlns:.*?".*?"', '', ret) + return re.sub(r' xmlns:.*?".*?"', '', ret.decode('utf8')) def helper(node): for child in node: @@ -205,7 +205,7 @@ def _CreateNodeHash(lines): assert False, 'Did not find end of node:\n' + '\n'.join(lines) # Insecure and truncated hash as it only needs to be unique vs. its neighbors. - return hashlib.md5('\n'.join(tag_lines)).hexdigest()[:8] + return hashlib.md5(('\n'.join(tag_lines)).encode('utf8')).hexdigest()[:8] def _IsSelfClosing(lines): diff --git a/chromium/build/android/gyp/util/manifest_utils_test.py b/chromium/build/android/gyp/util/manifest_utils_test.py index fe251960e7a..3423e5ba85e 100755 --- a/chromium/build/android/gyp/util/manifest_utils_test.py +++ b/chromium/build/android/gyp/util/manifest_utils_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2020 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/gyp/util/md5_check_test.py b/chromium/build/android/gyp/util/md5_check_test.py index 2169320ee54..f77e9715e64 100755 --- a/chromium/build/android/gyp/util/md5_check_test.py +++ b/chromium/build/android/gyp/util/md5_check_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/gyp/util/resource_utils_test.py b/chromium/build/android/gyp/util/resource_utils_test.py index 3026889460e..42a835672df 100755 --- a/chromium/build/android/gyp/util/resource_utils_test.py +++ b/chromium/build/android/gyp/util/resource_utils_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # coding: utf-8 # Copyright 2018 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/chromium/build/android/gyp/util/server_utils.py b/chromium/build/android/gyp/util/server_utils.py index c15f5f2cb91..e050ef65521 100644 --- a/chromium/build/android/gyp/util/server_utils.py +++ b/chromium/build/android/gyp/util/server_utils.py @@ -31,7 +31,7 @@ def MaybeRunCommand(name, argv, stamp_file): 'cmd': argv, 'cwd': os.getcwd(), 'stamp_file': stamp_file, - })) + }).encode('utf8')) except socket.error as e: # [Errno 111] Connection refused. Either the server has not been started # or the server is not currently accepting new connections. diff --git a/chromium/build/android/gyp/write_build_config.py b/chromium/build/android/gyp/write_build_config.py index 98ece1a7d62..5369a80a175 100755 --- a/chromium/build/android/gyp/write_build_config.py +++ b/chromium/build/android/gyp/write_build_config.py @@ -242,6 +242,14 @@ the current build. This type is used to describe target that wrap Java bytecode, either created by compiling sources, or providing them with a prebuilt jar. +* `deps_info['public_deps_configs']`: List of paths to the `.build_config` files +of *direct* dependencies of the current target which are exposed as part of the +current target's public API. This should be a subset of +deps_info['deps_configs']. + +* `deps_info['ignore_dependency_public_deps']`: If true, 'public_deps' will not +be collected from the current target's direct deps. + * `deps_info['unprocessed_jar_path']`: Path to the original .jar file for this target, before any kind of processing through Proguard or other tools. For most targets this is generated @@ -592,6 +600,23 @@ _ROOT_TYPES = ('android_apk', 'java_binary', 'java_annotation_processor', _RESOURCE_TYPES = ('android_assets', 'android_resources', 'system_java_library') +class OrderedSet(collections.OrderedDict): + # Value |parameter| is present to avoid presubmit warning due to different + # number of parameters from overridden method. + @staticmethod + def fromkeys(iterable, value=None): + out = OrderedSet() + out.update(iterable) + return out + + def add(self, key): + self[key] = True + + def update(self, iterable): + for v in iterable: + self.add(v) + + def _ExtractMarkdownDocumentation(input_text): """Extract Markdown documentation from a list of input strings lines. @@ -700,15 +725,21 @@ class Deps(object): return self._direct_deps_configs return DepsOfType(wanted_type, self._direct_deps_configs) + def DirectAndChildPublicDeps(self, wanted_type=None): + """Returns direct dependencies and dependencies exported via public_deps of + direct dependencies. + """ + dep_paths = set(self._direct_deps_config_paths) + for direct_dep in self._direct_deps_configs: + dep_paths.update(direct_dep.get('public_deps_configs', [])) + deps_list = [GetDepConfig(p) for p in dep_paths] + if wanted_type is None: + return deps_list + return DepsOfType(wanted_type, deps_list) + def AllConfigPaths(self): return self._all_deps_config_paths - def RemoveNonDirectDep(self, path): - if path in self._direct_deps_config_paths: - raise Exception('Cannot remove direct dep.') - self._all_deps_config_paths.remove(path) - self._all_deps_configs.remove(GetDepConfig(path)) - def GradlePrebuiltJarPaths(self): ret = [] @@ -902,6 +933,10 @@ def _AddJarMapping(jar_to_target, configs): jar_to_target[jar] = config['gn_target'] +def _CompareClasspathPriority(dep): + return 1 if dep.get('low_classpath_priority') else 0 + + def main(argv): parser = optparse.OptionParser() build_utils.AddDepfileOption(parser) @@ -950,6 +985,15 @@ def main(argv): help='Consider the assets as locale paks in BuildConfig.java') # java library options + + parser.add_option('--public-deps-configs', + help='GN list of config files of deps which are exposed as ' + 'part of the target\'s public API.') + parser.add_option( + '--ignore-dependency-public-deps', + action='store_true', + help='If true, \'public_deps\' will not be collected from the current ' + 'target\'s direct deps.') parser.add_option('--aar-path', help='Path to containing .aar file.') parser.add_option('--device-jar-path', help='Path to .jar for dexing.') parser.add_option('--host-jar-path', help='Path to .jar for java_binary.') @@ -973,6 +1017,11 @@ def main(argv): help='GYP-list of .jar files to include on the classpath when compiling, ' 'but not to include in the final binary.') parser.add_option( + '--low-classpath-priority', + action='store_true', + help='Indicates that the library should be placed at the end of the ' + 'classpath.') + parser.add_option( '--mergeable-android-manifests', help='GN-list of AndroidManifest.xml to include in manifest merging.') parser.add_option('--gradle-treat-as-prebuilt', action='store_true', @@ -1222,7 +1271,6 @@ def main(argv): direct_deps = deps.Direct() system_library_deps = deps.Direct('system_java_library') - direct_library_deps = deps.Direct('java_library') all_deps = deps.All() all_library_deps = deps.All('java_library') all_resources_deps = deps.All('android_resources') @@ -1377,6 +1425,9 @@ def main(argv): if options.unprocessed_jar_path: deps_info['unprocessed_jar_path'] = options.unprocessed_jar_path deps_info['interface_jar_path'] = options.interface_jar_path + if options.public_deps_configs: + deps_info['public_deps_configs'] = build_utils.ParseGnList( + options.public_deps_configs) if options.device_jar_path: deps_info['device_jar_path'] = options.device_jar_path if options.host_jar_path: @@ -1385,6 +1436,8 @@ def main(argv): deps_info['dex_path'] = options.dex_path if is_apk_or_module_target: all_dex_files.append(options.dex_path) + if options.low_classpath_priority: + deps_info['low_classpath_priority'] = True if options.type == 'android_apk': deps_info['apk_path'] = options.apk_path deps_info['incremental_apk_path'] = options.incremental_apk_path @@ -1529,21 +1582,36 @@ def main(argv): if is_java_target: + if options.ignore_dependency_public_deps: + classpath_direct_deps = deps.Direct() + classpath_direct_library_deps = deps.Direct('java_library') + else: + classpath_direct_deps = deps.DirectAndChildPublicDeps() + classpath_direct_library_deps = deps.DirectAndChildPublicDeps( + 'java_library') + # The classpath used to compile this target when annotation processors are # present. javac_classpath = set(c['unprocessed_jar_path'] - for c in direct_library_deps) + for c in classpath_direct_library_deps) # The classpath used to compile this target when annotation processors are # not present. These are also always used to know when a target needs to be # rebuilt. javac_interface_classpath = set(c['interface_jar_path'] - for c in direct_library_deps) + for c in classpath_direct_library_deps) + + # Preserve order of |all_library_deps|. Move low priority libraries to the + # end of the classpath. + all_library_deps_sorted_for_classpath = sorted( + all_library_deps[::-1], key=_CompareClasspathPriority) + # The classpath used for bytecode-rewritting. - javac_full_classpath = set(c['unprocessed_jar_path'] - for c in all_library_deps) + javac_full_classpath = OrderedSet.fromkeys( + c['unprocessed_jar_path'] + for c in all_library_deps_sorted_for_classpath) # The classpath used for error prone. - javac_full_interface_classpath = set(c['interface_jar_path'] - for c in all_library_deps) + javac_full_interface_classpath = OrderedSet.fromkeys( + c['interface_jar_path'] for c in all_library_deps_sorted_for_classpath) # Adding base module to classpath to compile against its R.java file if base_module_build_config: @@ -1556,7 +1624,7 @@ def main(argv): javac_interface_classpath.add( base_module_build_config['deps_info']['interface_jar_path']) - for dep in direct_deps: + for dep in classpath_direct_deps: if 'extra_classpath_jars' in dep: javac_classpath.update(dep['extra_classpath_jars']) javac_interface_classpath.update(dep['extra_classpath_jars']) @@ -1867,21 +1935,21 @@ def main(argv): ] config['javac']['processor_classes'] = [ c['main_class'] for c in processor_deps.Direct()] - deps_info['javac_full_classpath'] = sorted(javac_full_classpath) - deps_info['javac_full_interface_classpath'] = sorted( + deps_info['javac_full_classpath'] = list(javac_full_classpath) + deps_info['javac_full_interface_classpath'] = list( javac_full_interface_classpath) elif options.type == 'android_app_bundle': # bundles require javac_full_classpath to create .aab.jar.info and require # javac_full_interface_classpath for lint. - javac_full_classpath = set() - javac_full_interface_classpath = set() + javac_full_classpath = OrderedSet() + javac_full_interface_classpath = OrderedSet() for d in deps.Direct('android_app_bundle_module'): javac_full_classpath.update(d['javac_full_classpath']) javac_full_interface_classpath.update(d['javac_full_interface_classpath']) javac_full_classpath.add(d['unprocessed_jar_path']) javac_full_interface_classpath.add(d['interface_jar_path']) - deps_info['javac_full_classpath'] = sorted(javac_full_classpath) - deps_info['javac_full_interface_classpath'] = sorted( + deps_info['javac_full_classpath'] = list(javac_full_classpath) + deps_info['javac_full_interface_classpath'] = list( javac_full_interface_classpath) if options.type in ('android_apk', 'dist_jar', 'android_app_bundle_module', @@ -1998,9 +2066,8 @@ def main(argv): _AddJarMapping(jar_to_target, [base_module_build_config['deps_info']]) if options.tested_apk_config: _AddJarMapping(jar_to_target, [tested_apk_config]) - for jar, target in itertools.izip( - tested_apk_config['javac_full_classpath'], - tested_apk_config['javac_full_classpath_targets']): + for jar, target in zip(tested_apk_config['javac_full_classpath'], + tested_apk_config['javac_full_classpath_targets']): jar_to_target[jar] = target # Used by bytecode_processor to give better error message when missing diff --git a/chromium/build/android/gyp/write_native_libraries_java.py b/chromium/build/android/gyp/write_native_libraries_java.py index cb0c5d398ec..a0d99062016 100755 --- a/chromium/build/android/gyp/write_native_libraries_java.py +++ b/chromium/build/android/gyp/write_native_libraries_java.py @@ -95,7 +95,7 @@ def main(): with zipfile.ZipFile(f.name, 'w') as srcjar_file: build_utils.AddToZipHermetic( zip_file=srcjar_file, - zip_path='org/chromium/base/library_loader/NativeLibraries.java', + zip_path='org/chromium/build/NativeLibraries.java', data=NATIVE_LIBRARIES_TEMPLATE.format(**format_dict)) if options.depfile: diff --git a/chromium/build/android/host_heartbeat.py b/chromium/build/android/host_heartbeat.py index 6779a7cec59..4e11c5c2498 100755 --- a/chromium/build/android/host_heartbeat.py +++ b/chromium/build/android/host_heartbeat.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # # Copyright (c) 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/chromium/build/android/incremental_install/generate_android_manifest.py b/chromium/build/android/incremental_install/generate_android_manifest.py index 48dcc9979ea..e069dab80ef 100755 --- a/chromium/build/android/incremental_install/generate_android_manifest.py +++ b/chromium/build/android/incremental_install/generate_android_manifest.py @@ -97,9 +97,9 @@ def _ProcessManifest(path, arsc_package_name, disable_isolated_processes): ret = ElementTree.tostring(doc.getroot(), encoding='UTF-8') # Disable check for page-aligned native libraries. - ret = ret.replace('extractNativeLibs="false"', 'extractNativeLibs="true"') + ret = ret.replace(b'extractNativeLibs="false"', b'extractNativeLibs="true"') if disable_isolated_processes: - ret = ret.replace('isolatedProcess="true"', 'isolatedProcess="false"') + ret = ret.replace(b'isolatedProcess="true"', b'isolatedProcess="false"') return ret diff --git a/chromium/build/android/incremental_install/installer.py b/chromium/build/android/incremental_install/installer.py index 7329593b060..7ad80fe6efa 100755 --- a/chromium/build/android/incremental_install/installer.py +++ b/chromium/build/android/incremental_install/installer.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -228,7 +228,7 @@ def Install(device, install_json, apk=None, enable_device_cache=False, def check_device_configured(): target_sdk_version = int(apk.GetTargetSdkVersion()) - # Beta Q builds apply whitelist to targetSdk=28 as well. + # Beta Q builds apply allowlist to targetSdk=28 as well. if target_sdk_version >= 28 and device.build_version_sdk >= 28: # In P, there are two settings: # * hidden_api_policy_p_apps diff --git a/chromium/build/android/incremental_install/write_installer_json.py b/chromium/build/android/incremental_install/write_installer_json.py index 67a4e51568f..cf1d2d4c57a 100755 --- a/chromium/build/android/incremental_install/write_installer_json.py +++ b/chromium/build/android/incremental_install/write_installer_json.py @@ -60,7 +60,7 @@ def main(args): 'split_globs': options.split_globs, } - with build_utils.AtomicOutput(options.output_path) as f: + with build_utils.AtomicOutput(options.output_path, mode='w+') as f: json.dump(data, f, indent=2, sort_keys=True) diff --git a/chromium/build/android/list_class_verification_failures.py b/chromium/build/android/list_class_verification_failures.py index ff021804596..508e8312270 100755 --- a/chromium/build/android/list_class_verification_failures.py +++ b/chromium/build/android/list_class_verification_failures.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2018 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/list_java_targets.py b/chromium/build/android/list_java_targets.py index cd123c1a6b1..d0689a6e0c8 100755 --- a/chromium/build/android/list_java_targets.py +++ b/chromium/build/android/list_java_targets.py @@ -98,16 +98,20 @@ class _TargetEntry(object): def ninja_build_config_target(self): return self.ninja_target + '__build_config_crbug_908819' + @property + def build_config_path(self): + """Returns the filepath of the project's .build_config.""" + ninja_target = self.ninja_target + # Support targets at the root level. e.g. //:foo + if ninja_target[0] == ':': + ninja_target = ninja_target[1:] + subpath = ninja_target.replace(':', os.path.sep) + '.build_config' + return os.path.join(constants.GetOutDirectory(), 'gen', subpath) + def build_config(self): """Reads and returns the project's .build_config JSON.""" if not self._build_config: - ninja_target = self.ninja_target - # Support targets at the root level. e.g. //:foo - if ninja_target[0] == ':': - ninja_target = ninja_target[1:] - subpath = ninja_target.replace(':', os.path.sep) + '.build_config' - path = os.path.join('gen', subpath) - with open(os.path.join(constants.GetOutDirectory(), path)) as jsonfile: + with open(self.build_config_path) as jsonfile: self._build_config = json.load(jsonfile) return self._build_config @@ -141,6 +145,9 @@ def main(): parser.add_argument('--print-types', action='store_true', help='Print type of each target') + parser.add_argument('--print-build-config-paths', + action='store_true', + help='Print path to the .build_config of each target') parser.add_argument('--build', action='store_true', help='Build all .build_config files.') @@ -199,6 +206,8 @@ def main(): if args.print_types: to_print = f'{to_print}: {e.get_type()}' + elif args.print_build_config_paths: + to_print = f'{to_print}: {e.build_config_path}' print(to_print) diff --git a/chromium/build/android/native_flags/BUILD.gn b/chromium/build/android/native_flags/BUILD.gn new file mode 100644 index 00000000000..9c5be70ffd7 --- /dev/null +++ b/chromium/build/android/native_flags/BUILD.gn @@ -0,0 +1,37 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +if (current_toolchain == default_toolchain) { + import("//build/toolchain/toolchain.gni") + + # A toolchain that will capture compiler and linker arguments to a file. + toolchain("flagcapture") { + tool("cxx") { + cxx = rebase_path("argcapture.py", root_build_dir) + command = "$cxx {{output}} {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}}" + outputs = [ "{{root_out_dir}}/{{label_name}}.txt" ] + } + tool("solink") { + solink = rebase_path("argcapture.py", root_build_dir) + command = "$solink {{output}} {{ldflags}}" + outputs = [ "{{root_out_dir}}/{{label_name}}.txt" ] + } + tool("alink") { + command = "this-should-never-run" + outputs = [ "this-will-never-exist" ] + } + tool("stamp") { + command = stamp_command + description = stamp_description + } + } +} else if (current_toolchain == "//build/android/native_flags:flagcapture") { + # This will record flags from all default configs of the default toolchain. + source_set("default_ccflags") { + sources = [ "empty.cc" ] + } + shared_library("default_ldflags") { + no_default_deps = true + } +} diff --git a/chromium/build/android/native_flags/argcapture.py b/chromium/build/android/native_flags/argcapture.py new file mode 100755 index 00000000000..159b03ab887 --- /dev/null +++ b/chromium/build/android/native_flags/argcapture.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Writes arguments to a file.""" + +import sys + + +def main(): + with open(sys.argv[1], 'w') as f: + f.write('\n'.join(sys.argv[2:])) + f.write('\n') + + +if __name__ == '__main__': + main() diff --git a/chromium/build/android/native_flags/empty.cc b/chromium/build/android/native_flags/empty.cc new file mode 100644 index 00000000000..94aac140fbd --- /dev/null +++ b/chromium/build/android/native_flags/empty.cc @@ -0,0 +1,5 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This file just needs to exist to appease GN. diff --git a/chromium/build/android/play_services/__init__.py b/chromium/build/android/play_services/__init__.py deleted file mode 100644 index 50b23dff631..00000000000 --- a/chromium/build/android/play_services/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. diff --git a/chromium/build/android/play_services/preprocess.py b/chromium/build/android/play_services/preprocess.py deleted file mode 100755 index bb3424a80da..00000000000 --- a/chromium/build/android/play_services/preprocess.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2015 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -'''Prepares the Google Play services split client libraries before usage by -Chrome's build system. - -We need to preprocess Google Play services before using it in Chrome builds -mostly to remove unused resources (unsupported languages, unused drawables, -etc.) as proper resource shrinking is not yet supported by our build system. -(See https://crbug.com/636448) - -The script is meant to be used with an unpacked library repository. One can -be obtained by downloading the "extra-google-m2repository" from the Android SDK -Manager and extracting the AARs from the desired version as the following -structure: - - REPOSITORY_DIR - +-- CLIENT_1 - | +-- <content of the first AAR file> - +-- CLIENT_2 - +-- etc. - -The output will follow the same structure, with fewer resource files, in the -provided output directory. -''' - -import argparse -import glob -import itertools -import os -import shutil -import stat -import sys -import tempfile -import textwrap -import zipfile - -sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir)) -from play_services import utils -from pylib.utils import argparse_utils - - -def main(): - parser = argparse.ArgumentParser(description=( - "Prepares the Google Play services split client libraries before usage " - "by Chrome's build system. See the script's documentation for more a " - "detailed help.")) - argparse_utils.CustomHelpAction.EnableFor(parser) - required_args = parser.add_argument_group('required named arguments') - required_args.add_argument('-r', - '--repository', - help=('the Google Play services repository ' - 'location'), - required=True, - metavar='FILE') - required_args.add_argument('-d', - '--root-dir', - help='the directory which GN considers the root', - required=True, - metavar='FILE') - required_args.add_argument('-o', - '--out-dir', - help='the output directory', - required=True, - metavar='FILE') - required_args.add_argument('-g', - '--gni-out-file', - help='the GN output file', - required=True, - metavar='FILE') - required_args.add_argument('-c', - '--config-file', - help='the config file path', - required=True, - metavar='FILE') - parser.add_argument('--config-help', - action='custom_help', - custom_help_text=utils.ConfigParser.__doc__, - help='show the configuration file format help') - - args = parser.parse_args() - - return ProcessGooglePlayServices(args.repository, - args.root_dir, - args.out_dir, - args.gni_out_file, - args.config_file) - - -def ProcessGooglePlayServices( - repo, root_dir, out_dir, gni_out_file, config_path): - config = utils.ConfigParser(config_path) - - tmp_root = tempfile.mkdtemp() - try: - tmp_paths = _SetupTempDir(tmp_root) - _ImportFromExtractedRepo(config, tmp_paths, repo) - _ProcessResources(config, tmp_paths, repo) - _CopyToOutput(tmp_paths, out_dir) - _EnumerateProguardFiles(root_dir, out_dir, gni_out_file) - _UpdateVersionInConfig(config, tmp_paths) - finally: - shutil.rmtree(tmp_root) - - return 0 - - -def _SetupTempDir(tmp_root): - tmp_paths = { - 'root': tmp_root, - 'imported_clients': os.path.join(tmp_root, 'imported_clients'), - 'extracted_jars': os.path.join(tmp_root, 'jar'), - 'combined_jar': os.path.join(tmp_root, 'google-play-services.jar'), - } - os.mkdir(tmp_paths['imported_clients']) - os.mkdir(tmp_paths['extracted_jars']) - - return tmp_paths - - -def _MakeWritable(dir_path): - for root, dirs, files in os.walk(dir_path): - for path in itertools.chain(dirs, files): - st = os.stat(os.path.join(root, path)) - os.chmod(os.path.join(root, path), st.st_mode | stat.S_IWUSR) - - -# E.g. turn "base_1p" into "base" -def _RemovePartySuffix(client): - return client[:-3] if client[-3:] == '_1p' else client - - -def _ImportFromExtractedRepo(config, tmp_paths, repo): - # Import the clients - try: - for client in config.clients: - client_out_dir = os.path.join(tmp_paths['imported_clients'], client) - shutil.copytree(os.path.join(repo, client), client_out_dir) - finally: - _MakeWritable(tmp_paths['imported_clients']) - - -def _ProcessResources(config, tmp_paths, repo): - LOCALIZED_VALUES_BASE_NAME = 'values-' - locale_whitelist = set(config.locale_whitelist) - - # The directory structure here is: - # <imported_clients temp dir>/<client name>_1p/res/<res type>/<res file>.xml - for client_dir in os.listdir(tmp_paths['imported_clients']): - client_prefix = _RemovePartySuffix(client_dir) + '_' - - res_path = os.path.join(tmp_paths['imported_clients'], client_dir, 'res') - if not os.path.isdir(res_path): - continue - - for res_type in os.listdir(res_path): - res_type_path = os.path.join(res_path, res_type) - - if res_type.startswith('drawable'): - shutil.rmtree(res_type_path) - continue - - if res_type.startswith(LOCALIZED_VALUES_BASE_NAME): - dir_locale = res_type[len(LOCALIZED_VALUES_BASE_NAME):] - if dir_locale not in locale_whitelist: - shutil.rmtree(res_type_path) - continue - - if res_type.startswith('values'): - # Beginning with v3, resource file names are not necessarily unique, - # and would overwrite each other when merged at build time. Prefix each - # "values" resource file with its client name. - for res_file in os.listdir(res_type_path): - os.rename(os.path.join(res_type_path, res_file), - os.path.join(res_type_path, client_prefix + res_file)) - - # Reimport files from the whitelist. - for res_path in config.resource_whitelist: - for whitelisted_file in glob.glob(os.path.join(repo, res_path)): - resolved_file = os.path.relpath(whitelisted_file, repo) - rebased_res = os.path.join(tmp_paths['imported_clients'], resolved_file) - - if not os.path.exists(os.path.dirname(rebased_res)): - os.makedirs(os.path.dirname(rebased_res)) - - try: - shutil.copy(os.path.join(repo, whitelisted_file), rebased_res) - finally: - _MakeWritable(rebased_res) - - -def _CopyToOutput(tmp_paths, out_dir): - shutil.rmtree(out_dir, ignore_errors=True) - shutil.copytree(tmp_paths['imported_clients'], out_dir) - - -# Write a GN file containing a list of each GMS client's proguard file (if any). -def _EnumerateProguardFiles(root_dir, out_dir, gni_path): - gni_dir = os.path.dirname(gni_path) - gni_template = textwrap.dedent('''\ - # Copyright 2017 The Chromium Authors. All rights reserved. - # Use of this source code is governed by a BSD-style license that can be - # found in the LICENSE file. - - # This file generated by {script} - gms_proguard_configs = [ - {body} - ] - ''') - - gni_lines = [] - for client_dir in os.listdir(out_dir): - proguard_path = os.path.join( - out_dir, client_dir, 'proguard.txt') - if os.path.exists(proguard_path): - rooted_path = os.path.relpath(proguard_path, root_dir) - gni_lines.append(' "//{}",'.format(rooted_path)) - gni_lines.sort() - - gni_text = gni_template.format( - script=os.path.relpath(sys.argv[0], gni_dir), - body='\n'.join(gni_lines)) - - with open(gni_path, 'w') as gni_file: - gni_file.write(gni_text) - - -def _UpdateVersionInConfig(config, tmp_paths): - version_xml_path = os.path.join(tmp_paths['imported_clients'], - config.version_xml_path) - play_services_full_version = utils.GetVersionNumberFromLibraryResources( - version_xml_path) - config.UpdateVersionNumber(play_services_full_version) - - -def _ExtractAll(zip_path, out_path): - with zipfile.ZipFile(zip_path, 'r') as zip_file: - zip_file.extractall(out_path) - -if __name__ == '__main__': - sys.exit(main()) diff --git a/chromium/build/android/play_services/utils.py b/chromium/build/android/play_services/utils.py deleted file mode 100644 index 76b3679957c..00000000000 --- a/chromium/build/android/play_services/utils.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -''' -Utility functions for all things related to manipulating google play services -related files. -''' - -import argparse -import filecmp -import json -import os -import re - - -_XML_VERSION_NUMBER_PATTERN = re.compile( - r'<integer name="google_play_services_version">(\d+)<\/integer>') - - -class DefaultsRawHelpFormatter(argparse.ArgumentDefaultsHelpFormatter, - argparse.RawDescriptionHelpFormatter): - ''' - Combines the features of RawDescriptionHelpFormatter and - ArgumentDefaultsHelpFormatter, providing defaults for the arguments and raw - text for the description. - ''' - pass - - -class ConfigParser(object): - '''Reads and writes the configuration files for play services related scripts - - The configuration files are JSON files. Here is the data they are expected - to contain: - - - version_number - Number. Mirrors @integer/google_play_services_version from the library. - Example: 815000 - - - sdk_version - Version of the Play Services SDK to retrieve, when preprocessing the - library from a maven/gradle repository. - Example: "8.1.0" - - - clients - List of strings. Name of the clients (or play services modules) to - include when preprocessing the library. - Example: ["play-services-base", "play-services-cast"] - - - version_xml_path - String. Path to the version.xml string describing the current version. - Should be relative to the library base directory - Example: "res/values/version.xml" - - - locale_whitelist - List of strings. List of locales to keep from the resources. Can be - obtained by generating an android build and looking at the content of - `out/Debug/gen/chrome/java/res`; or looking at the android section in - `//chrome/app/generated_resources.grd` - Example: ["am", "ar", "bg", "ca", "cs"] - - - resource_whitelist - List of strings. List of resource files to explicitely keep in the final - output. Use it to keep drawables for example, as we currently remove them - all. - Example: ["play-services-base/res/drawables/foobar.xml"] - ''' - _VERSION_NUMBER_KEY = 'version_number' - - def __init__(self, path): - self.path = path - self._data = {} - - with open(path, 'r') as stream: - self._data = json.load(stream) - - @property - def version_number(self): - return self._data.get(self._VERSION_NUMBER_KEY) - - @property - def sdk_version(self): - return self._data.get('sdk_version') - - @property - def clients(self): - return self._data.get('clients') or [] - - @property - def version_xml_path(self): - return self._data.get('version_xml_path') - - @property - def locale_whitelist(self): - return self._data.get('locale_whitelist') or [] - - @property - def resource_whitelist(self): - return self._data.get('resource_whitelist') or [] - - def UpdateVersionNumber(self, new_version_number): - '''Updates the version number and saves it in the configuration file. ''' - - with open(self.path, 'w') as stream: - self._data[self._VERSION_NUMBER_KEY] = new_version_number - stream.write(DumpTrimmedJson(self._data)) - - -def DumpTrimmedJson(json_data): - ''' - Default formatting when dumping json to string has trailing spaces and lacks - a new line at the end. This function fixes that. - ''' - - out = json.dumps(json_data, sort_keys=True, indent=2) - out = out.replace(' ' + os.linesep, os.linesep) - return out + os.linesep - - -def FileEquals(expected_file, actual_file): - ''' - Returns whether the two files are equal. Returns False if any of the files - doesn't exist. - ''' - - if not os.path.isfile(actual_file) or not os.path.isfile(expected_file): - return False - return filecmp.cmp(expected_file, actual_file) - - -def GetVersionNumberFromLibraryResources(version_xml): - ''' - Extracts a Google Play services version number from its version.xml file. - ''' - - with open(version_xml, 'r') as version_file: - version_file_content = version_file.read() - - match = _XML_VERSION_NUMBER_PATTERN.search(version_file_content) - if not match: - raise AttributeError('A value for google_play_services_version was not ' - 'found in ' + version_xml) - return int(match.group(1)) diff --git a/chromium/build/android/provision_devices.py b/chromium/build/android/provision_devices.py index e03c664a68c..5fb4d93d490 100755 --- a/chromium/build/android/provision_devices.py +++ b/chromium/build/android/provision_devices.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # # Copyright (c) 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/chromium/build/android/pylib/base/result_sink.py b/chromium/build/android/pylib/base/result_sink.py index 936e1f35761..76758a06bb3 100644 --- a/chromium/build/android/pylib/base/result_sink.py +++ b/chromium/build/android/pylib/base/result_sink.py @@ -15,7 +15,7 @@ MAX_REPORT_LEN = 4 * 1024 # Maps base_test_results to the luci test-result.proto. # https://godoc.org/go.chromium.org/luci/resultdb/proto/v1#TestStatus RESULT_MAP = { - base_test_result.ResultType.UNKNOWN: 'STATUS_UNSPECIFIED', + base_test_result.ResultType.UNKNOWN: 'ABORT', base_test_result.ResultType.PASS: 'PASS', base_test_result.ResultType.FAIL: 'FAIL', base_test_result.ResultType.CRASH: 'CRASH', @@ -58,7 +58,8 @@ class ResultSinkClient(object): 'Authorization': 'ResultSink %s' % context['auth_token'], } - def Post(self, test_id, status, test_log, test_file, artifacts=None): + def Post(self, test_id, status, duration, test_log, test_file, + artifacts=None): """Uploads the test result to the ResultSink server. This assumes that the rdb stream has been called already and that @@ -67,6 +68,7 @@ class ResultSinkClient(object): Args: test_id: A string representing the test's name. status: A string representing if the test passed, failed, etc... + duration: An int representing time in ms. test_log: A string representing the test's output. test_file: A string representing the file location of the test. artifacts: An optional dict of artifacts to attach to the test. @@ -77,7 +79,7 @@ class ResultSinkClient(object): assert status in RESULT_MAP expected = status in (base_test_result.ResultType.PASS, base_test_result.ResultType.SKIP) - status = RESULT_MAP[status] + result_db_status = RESULT_MAP[status] # Slightly smaller to allow addition of <pre> tags and message. report_check_size = MAX_REPORT_LEN - 45 @@ -89,14 +91,29 @@ class ResultSinkClient(object): test_log_formatted = '<pre>' + test_log_escaped + '</pre>' tr = { - 'expected': expected, - 'status': status, - 'summaryHtml': test_log_formatted, - 'tags': [{ - 'key': 'test_name', - 'value': test_id, - }], - 'testId': test_id, + # Duration must be formatted to avoid scientific notation in case + # number is too small or too large. Result_db takes seconds, not ms. + 'duration': + '%.9fs' % (duration / 1000.0), + 'expected': + expected, + 'status': + result_db_status, + 'summaryHtml': + test_log_formatted, + 'tags': [ + { + 'key': 'test_name', + 'value': test_id, + }, + { + # Status before getting mapped to result_db statuses. + 'key': 'android_test_runner_status', + 'value': status, + } + ], + 'testId': + test_id, } artifacts = artifacts or {} if len(test_log_escaped) > report_check_size: diff --git a/chromium/build/android/pylib/gtest/gtest_test_instance.py b/chromium/build/android/pylib/gtest/gtest_test_instance.py index 4b751105023..294e2dbb5f4 100644 --- a/chromium/build/android/pylib/gtest/gtest_test_instance.py +++ b/chromium/build/android/pylib/gtest/gtest_test_instance.py @@ -101,8 +101,9 @@ _RE_TEST_STATUS = re.compile( # Crash detection constants. _RE_TEST_ERROR = re.compile(r'FAILURES!!! Tests run: \d+,' r' Failures: \d+, Errors: 1') -_RE_TEST_CURRENTLY_RUNNING = re.compile(r'\[ERROR:.*?\]' - r' Currently running: (.*)') +_RE_TEST_CURRENTLY_RUNNING = re.compile( + r'\[ERROR:.*?\] Currently running: (.*)') +_RE_TEST_DCHECK_FATAL = re.compile(r'\[.*:FATAL:.*\] (.*)') _RE_DISABLED = re.compile(r'DISABLED_') _RE_FLAKY = re.compile(r'FLAKY_') @@ -174,10 +175,15 @@ def ParseGTestOutput(output, symbolizer, device_abi): def handle_possibly_unknown_test(): if test_name is not None: - results.append(base_test_result.BaseTestResult( - TestNameWithoutDisabledPrefix(test_name), - fallback_result_type or base_test_result.ResultType.UNKNOWN, - duration, log=symbolize_stack_and_merge_with_log())) + results.append( + base_test_result.BaseTestResult( + TestNameWithoutDisabledPrefix(test_name), + # If we get here, that means we started a test, but it did not + # produce a definitive test status output, so assume it crashed. + # crbug/1191716 + fallback_result_type or base_test_result.ResultType.CRASH, + duration, + log=symbolize_stack_and_merge_with_log())) for l in output: matcher = _RE_TEST_STATUS.match(l) @@ -202,10 +208,15 @@ def ParseGTestOutput(output, symbolizer, device_abi): duration = int(matcher.group(3)) if matcher.group(3) else 0 else: - # Needs another matcher here to match crashes, like those of DCHECK. - matcher = _RE_TEST_CURRENTLY_RUNNING.match(l) - if matcher: - test_name = matcher.group(1) + # Can possibly add more matchers, such as different results from DCHECK. + currently_running_matcher = _RE_TEST_CURRENTLY_RUNNING.match(l) + dcheck_matcher = _RE_TEST_DCHECK_FATAL.match(l) + + if currently_running_matcher: + test_name = currently_running_matcher.group(1) + result_type = base_test_result.ResultType.CRASH + duration = 0 # Don't know. + elif dcheck_matcher: result_type = base_test_result.ResultType.CRASH duration = 0 # Don't know. diff --git a/chromium/build/android/pylib/gtest/gtest_test_instance_test.py b/chromium/build/android/pylib/gtest/gtest_test_instance_test.py index 77a5556a1fb..f3e85d04f98 100755 --- a/chromium/build/android/pylib/gtest/gtest_test_instance_test.py +++ b/chromium/build/android/pylib/gtest/gtest_test_instance_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -137,6 +137,17 @@ class GtestTestInstanceTests(unittest.TestCase): self.assertEquals(0, actual[0].GetDuration()) self.assertEquals(base_test_result.ResultType.CRASH, actual[0].GetType()) + def testParseGTestOutput_fatalDcheck(self): + raw_output = [ + '[ RUN ] FooTest.Bar', + '[0324/183029.116334:FATAL:test_timeouts.cc(103)] Check failed: !init', + ] + actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) + self.assertEquals(1, len(actual)) + self.assertEquals('FooTest.Bar', actual[0].GetName()) + self.assertEquals(0, actual[0].GetDuration()) + self.assertEquals(base_test_result.ResultType.CRASH, actual[0].GetType()) + def testParseGTestOutput_unknown(self): raw_output = [ '[ RUN ] FooTest.Bar', @@ -145,7 +156,7 @@ class GtestTestInstanceTests(unittest.TestCase): self.assertEquals(1, len(actual)) self.assertEquals('FooTest.Bar', actual[0].GetName()) self.assertEquals(0, actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.UNKNOWN, actual[0].GetType()) + self.assertEquals(base_test_result.ResultType.CRASH, actual[0].GetType()) def testParseGTestOutput_nonterminalUnknown(self): raw_output = [ @@ -158,7 +169,7 @@ class GtestTestInstanceTests(unittest.TestCase): self.assertEquals('FooTest.Bar', actual[0].GetName()) self.assertEquals(0, actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.UNKNOWN, actual[0].GetType()) + self.assertEquals(base_test_result.ResultType.CRASH, actual[0].GetType()) self.assertEquals('FooTest.Baz', actual[1].GetName()) self.assertEquals(1, actual[1].GetDuration()) diff --git a/chromium/build/android/pylib/instrumentation/instrumentation_parser_test.py b/chromium/build/android/pylib/instrumentation/instrumentation_parser_test.py index 092d10fc938..d664455c783 100755 --- a/chromium/build/android/pylib/instrumentation/instrumentation_parser_test.py +++ b/chromium/build/android/pylib/instrumentation/instrumentation_parser_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/pylib/instrumentation/instrumentation_test_instance.py b/chromium/build/android/pylib/instrumentation/instrumentation_test_instance.py index ae70fbfbd3a..5493c365f0c 100644 --- a/chromium/build/android/pylib/instrumentation/instrumentation_test_instance.py +++ b/chromium/build/android/pylib/instrumentation/instrumentation_test_instance.py @@ -41,16 +41,6 @@ _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS = [ _VALID_ANNOTATIONS = set(_DEFAULT_ANNOTATIONS + _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS) -_EXTRA_DRIVER_TEST_LIST = ( - 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestList') -_EXTRA_DRIVER_TEST_LIST_FILE = ( - 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestListFile') -_EXTRA_DRIVER_TARGET_PACKAGE = ( - 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetPackage') -_EXTRA_DRIVER_TARGET_CLASS = ( - 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetClass') -_EXTRA_TIMEOUT_SCALE = ( - 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TimeoutScale') _TEST_LIST_JUNIT4_RUNNERS = [ 'org.chromium.base.test.BaseChromiumAndroidJUnitRunner'] @@ -78,7 +68,6 @@ _BUNDLE_STACK_ID = 'stack' # The ID of the bundle value Chrome uses to report the test duration. _BUNDLE_DURATION_ID = 'duration_ms' - class MissingSizeAnnotationError(test_exception.TestException): def __init__(self, class_name): super(MissingSizeAnnotationError, self).__init__(class_name + @@ -532,17 +521,13 @@ class InstrumentationTestInstance(test_instance.TestInstance): self._use_apk_under_test_flags_file = False self._initializeFlagAttributes(args) - self._driver_apk = None - self._driver_package = None - self._driver_name = None - self._initializeDriverAttributes() - self._screenshot_dir = None self._timeout_scale = None self._wait_for_java_debugger = None self._initializeTestControlAttributes(args) self._coverage_directory = None + self._jacoco_coverage_type = None self._initializeTestCoverageAttributes(args) self._store_tombstones = False @@ -731,17 +716,6 @@ class InstrumentationTestInstance(test_instance.TestInstance): not args.coverage_dir): self._flags.append('--strict-mode=' + args.strict_mode) - def _initializeDriverAttributes(self): - self._driver_apk = os.path.join( - constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR, - 'OnDeviceInstrumentationDriver.apk') - if os.path.exists(self._driver_apk): - driver_apk = apk_helper.ApkHelper(self._driver_apk) - self._driver_package = driver_apk.GetPackageName() - self._driver_name = driver_apk.GetInstrumentationName() - else: - self._driver_apk = None - def _initializeTestControlAttributes(self, args): self._screenshot_dir = args.screenshot_dir self._timeout_scale = args.timeout_scale or 1 @@ -749,6 +723,12 @@ class InstrumentationTestInstance(test_instance.TestInstance): def _initializeTestCoverageAttributes(self, args): self._coverage_directory = args.coverage_dir + if ("Batch", "UnitTests") in self._annotations and ( + "Batch", "UnitTests") not in self._excluded_annotations: + self._jacoco_coverage_type = "unit_tests_only" + elif ("Batch", "UnitTests") not in self._annotations and ( + "Batch", "UnitTests") in self._excluded_annotations: + self._jacoco_coverage_type = "unit_tests_excluded" def _initializeLogAttributes(self, args): self._enable_java_deobfuscation = args.enable_java_deobfuscation @@ -815,18 +795,6 @@ class InstrumentationTestInstance(test_instance.TestInstance): return self._coverage_directory @property - def driver_apk(self): - return self._driver_apk - - @property - def driver_package(self): - return self._driver_package - - @property - def driver_name(self): - return self._driver_name - - @property def edit_shared_prefs(self): return self._edit_shared_prefs @@ -839,6 +807,10 @@ class InstrumentationTestInstance(test_instance.TestInstance): return self._flags @property + def jacoco_coverage_type(self): + return self._jacoco_coverage_type + + @property def junit3_runner_class(self): return self._junit3_runner_class @@ -1049,23 +1021,6 @@ class InstrumentationTestInstance(test_instance.TestInstance): new_tests.append(parameterized_t) return tests + new_tests - def GetDriverEnvironmentVars( - self, test_list=None, test_list_file_path=None): - env = { - _EXTRA_DRIVER_TARGET_PACKAGE: self.test_package, - _EXTRA_DRIVER_TARGET_CLASS: self.junit3_runner_class, - _EXTRA_TIMEOUT_SCALE: self._timeout_scale, - } - - if test_list: - env[_EXTRA_DRIVER_TEST_LIST] = ','.join(test_list) - - if test_list_file_path: - env[_EXTRA_DRIVER_TEST_LIST_FILE] = ( - os.path.basename(test_list_file_path)) - - return env - @staticmethod def ParseAmInstrumentRawOutput(raw_output): return ParseAmInstrumentRawOutput(raw_output) diff --git a/chromium/build/android/pylib/instrumentation/instrumentation_test_instance_test.py b/chromium/build/android/pylib/instrumentation/instrumentation_test_instance_test.py index fdb4114a63d..77918bb3ea5 100755 --- a/chromium/build/android/pylib/instrumentation/instrumentation_test_instance_test.py +++ b/chromium/build/android/pylib/instrumentation/instrumentation_test_instance_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -33,7 +33,6 @@ class InstrumentationTestInstanceTest(unittest.TestCase): mock.patch('%s._initializeDataDependencyAttributes' % c)), ( mock.patch('%s._initializeTestFilterAttributes' %c)), ( mock.patch('%s._initializeFlagAttributes' % c)), ( - mock.patch('%s._initializeDriverAttributes' % c)), ( mock.patch('%s._initializeTestControlAttributes' % c)), ( mock.patch('%s._initializeTestCoverageAttributes' % c)), ( mock.patch('%s._initializeSkiaGoldAttributes' % c)): diff --git a/chromium/build/android/pylib/local/device/local_device_gtest_run.py b/chromium/build/android/pylib/local/device/local_device_gtest_run.py index d9636de8f2c..258a309151b 100644 --- a/chromium/build/android/pylib/local/device/local_device_gtest_run.py +++ b/chromium/build/android/pylib/local/device/local_device_gtest_run.py @@ -6,6 +6,7 @@ import contextlib import collections import itertools import logging +import math import os import posixpath import subprocess @@ -53,9 +54,6 @@ _EXTRA_TEST_LIST = ( _SECONDS_TO_NANOS = int(1e9) -# The amount of time a test executable may run before it gets killed. -_TEST_TIMEOUT_SECONDS = 30*60 - # Tests that use SpawnedTestServer must run the LocalTestServerSpawner on the # host machine. # TODO(jbudorick): Move this up to the test instance if the net test server is @@ -116,6 +114,16 @@ def _ExtractTestsFromFilter(gtest_filter): return patterns +def _GetDeviceTimeoutMultiplier(): + # Emulated devices typically run 20-150x slower than real-time. + # Give a way to control this through the DEVICE_TIMEOUT_MULTIPLIER + # environment variable. + multiplier = os.getenv("DEVICE_TIMEOUT_MULTIPLIER") + if multiplier: + return int(multiplier) + return 1 + + def _MergeCoverageFiles(coverage_dir, profdata_dir): """Merge coverage data files. @@ -435,7 +443,10 @@ class _ExeDelegate(object): pass def Clear(self, device): - device.KillAll(self._exe_file_name, blocking=True, timeout=30, quiet=True) + device.KillAll(self._exe_file_name, + blocking=True, + timeout=30 * _GetDeviceTimeoutMultiplier(), + quiet=True) class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun): @@ -491,7 +502,7 @@ class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun): # Some gtest suites, e.g. unit_tests, have data dependencies that # can take longer than the default timeout to push. See # crbug.com/791632 for context. - timeout=600) + timeout=600 * math.ceil(_GetDeviceTimeoutMultiplier() / 10)) if not host_device_tuples: dev.RemovePath(device_root, force=True, recursive=True, rename=True) dev.RunShellCommand(['mkdir', '-p', device_root], check_return=True) @@ -569,14 +580,9 @@ class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun): # Delete suspect testcase from tests. tests = [test for test in tests if not test in self._crashes] - batch_size = self._test_instance.test_launcher_batch_limit + max_shard_size = self._test_instance.test_launcher_batch_limit - for i in xrange(0, device_count): - unbounded_shard = tests[i::device_count] - shards += [ - unbounded_shard[j:j + batch_size] - for j in xrange(0, len(unbounded_shard), batch_size) - ] + shards.extend(self._PartitionTests(tests, device_count, max_shard_size)) return shards #override @@ -595,7 +601,7 @@ class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun): @local_device_environment.handle_shard_failures_with( on_failure=self._env.DenylistDevice) def list_tests(dev): - timeout = 30 + timeout = 30 * _GetDeviceTimeoutMultiplier() retries = 1 if self._test_instance.wait_for_java_debugger: timeout = None @@ -689,8 +695,9 @@ class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun): else: desc = hash(tuple(test)) - stream_name = 'logcat_%s_%s_%s' % ( - desc, time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()), device.serial) + stream_name = 'logcat_%s_shard%s_%s_%s' % ( + desc, self._test_instance.external_shard_index, + time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()), device.serial) logcat_file = None logmon = None @@ -714,8 +721,9 @@ class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun): #override def _RunTest(self, device, test): # Run the test. - timeout = (self._test_instance.shard_timeout - * self.GetTool(device).GetTimeoutScale()) + timeout = (self._test_instance.shard_timeout * + self.GetTool(device).GetTimeoutScale() * + _GetDeviceTimeoutMultiplier()) if self._test_instance.wait_for_java_debugger: timeout = None if self._test_instance.store_tombstones: diff --git a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py index 135f1723b5a..37467f45da7 100644 --- a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py +++ b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py @@ -64,15 +64,23 @@ _WPR_GO_LINUX_X86_64_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, _TAG = 'test_runner_py' TIMEOUT_ANNOTATIONS = [ - ('Manual', 10 * 60 * 60), - ('IntegrationTest', 30 * 60), - ('External', 10 * 60), - ('EnormousTest', 10 * 60), - ('LargeTest', 5 * 60), - ('MediumTest', 3 * 60), - ('SmallTest', 1 * 60), + ('Manual', 10 * 60 * 60), + ('IntegrationTest', 10 * 60), + ('External', 10 * 60), + ('EnormousTest', 5 * 60), + ('LargeTest', 2 * 60), + ('MediumTest', 30), + ('SmallTest', 10), ] +# Account for Instrumentation and process init overhead. +FIXED_TEST_TIMEOUT_OVERHEAD = 60 + +# 30 minute max timeout for an instrumentation invocation to avoid shard +# timeouts when tests never finish. The shard timeout is currently 60 minutes, +# so this needs to be less than that. +MAX_BATCH_TEST_TIMEOUT = 30 * 60 + LOGCAT_FILTERS = ['*:e', 'chromium:v', 'cr_*:v', 'DEBUG:I', 'StrictMode:D', '%s:I' % _TAG] @@ -98,7 +106,8 @@ WPR_RECORD_REPLAY_TEST_FEATURE_ANNOTATION = 'WPRRecordReplayTest' _DEVICE_GOLD_DIR = 'skia_gold' # A map of Android product models to SDK ints. RENDER_TEST_MODEL_SDK_CONFIGS = { - 'Nexus 5X': [23], + # Android x86 emulator. + 'Android SDK built for x86': [23], } _TEST_BATCH_MAX_GROUP_SIZE = 256 @@ -538,17 +547,20 @@ class LocalDeviceInstrumentationTestRun( flags_to_add = [] test_timeout_scale = None if self._test_instance.coverage_directory: - coverage_basename = '%s.exec' % ( - '%s_%s_group' % (test[0]['class'], test[0]['method']) if isinstance( - test, list) else '%s_%s' % (test['class'], test['method'])) + coverage_basename = '%s' % ('%s_%s_group' % + (test[0]['class'], test[0]['method']) + if isinstance(test, list) else '%s_%s' % + (test['class'], test['method'])) + if self._test_instance.jacoco_coverage_type: + coverage_basename += "_" + self._test_instance.jacoco_coverage_type extras['coverage'] = 'true' coverage_directory = os.path.join( device.GetExternalStoragePath(), 'chrome', 'test', 'coverage') if not device.PathExists(coverage_directory): device.RunShellCommand(['mkdir', '-p', coverage_directory], check_return=True) - coverage_device_file = os.path.join( - coverage_directory, coverage_basename) + coverage_device_file = os.path.join(coverage_directory, coverage_basename, + '.exec') extras['coverageFile'] = coverage_device_file # Save screenshot if screenshot dir is specified (save locally) or if # a GS bucket is passed (save in cloud). @@ -584,7 +596,8 @@ class LocalDeviceInstrumentationTestRun( test_name = instrumentation_test_instance.GetTestName(test[0]) + '_batch' extras['class'] = ','.join(test_names) test_display_name = test_name - timeout = sum(timeouts) + timeout = min(MAX_BATCH_TEST_TIMEOUT, + FIXED_TEST_TIMEOUT_OVERHEAD + sum(timeouts)) else: assert test['is_junit4'] test_name = instrumentation_test_instance.GetTestName(test) @@ -593,8 +606,8 @@ class LocalDeviceInstrumentationTestRun( extras['class'] = test_name if 'flags' in test and test['flags']: flags_to_add.extend(test['flags']) - timeout = self._GetTimeoutFromAnnotations( - test['annotations'], test_display_name) + timeout = FIXED_TEST_TIMEOUT_OVERHEAD + self._GetTimeoutFromAnnotations( + test['annotations'], test_display_name) test_timeout_scale = self._GetTimeoutScaleFromAnnotations( test['annotations']) @@ -921,10 +934,9 @@ class LocalDeviceInstrumentationTestRun( @contextlib.contextmanager def _ArchiveLogcat(self, device, test_name): - stream_name = 'logcat_%s_%s_%s' % ( - test_name.replace('#', '.'), - time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()), - device.serial) + stream_name = 'logcat_%s_%s_%s' % (test_name.replace( + '#', '.'), time.strftime('%Y%m%dT%H%M%S-UTC', + time.gmtime()), device.serial) logcat_file = None logmon = None diff --git a/chromium/build/android/pylib/local/device/local_device_test_run.py b/chromium/build/android/pylib/local/device/local_device_test_run.py index 05128c6e0c5..6fa0af7ceab 100644 --- a/chromium/build/android/pylib/local/device/local_device_test_run.py +++ b/chromium/build/android/pylib/local/device/local_device_test_run.py @@ -70,6 +70,10 @@ class LocalDeviceTestRun(test_run.TestRun): SetAppCompatibilityFlagsIfNecessary(self._installed_packages, dev) consecutive_device_errors = 0 for test in tests: + if not test: + logging.warning('No tests in shared. Continuing.') + tests.test_completed() + continue if exit_now.isSet(): thread.exit() @@ -247,16 +251,92 @@ class LocalDeviceTestRun(test_run.TestRun): raise InvalidShardingSettings(shard_index, total_shards) sharded_tests = [] - for t in self._GroupTests(tests): - if (hash(self._GetUniqueTestName(t[0] if isinstance(t, list) else t)) % - total_shards == shard_index): - if isinstance(t, list): - sharded_tests.extend(t) - else: - sharded_tests.append(t) + # Group tests by tests that should run in the same test invocation - either + # unit tests or batched tests. + grouped_tests = self._GroupTests(tests) + + # Partition grouped tests approximately evenly across shards. + partitioned_tests = self._PartitionTests(grouped_tests, total_shards, + float('inf')) + if len(partitioned_tests) <= shard_index: + return [] + for t in partitioned_tests[shard_index]: + if isinstance(t, list): + sharded_tests.extend(t) + else: + sharded_tests.append(t) return sharded_tests + # Partition tests evenly into |num_desired_partitions| partitions where + # possible. However, many constraints make partitioning perfectly impossible. + # If the max_partition_size isn't large enough, extra partitions may be + # created (infinite max size should always return precisely the desired + # number of partitions). Even if the |max_partition_size| is technically large + # enough to hold all of the tests in |num_desired_partitions|, we attempt to + # keep test order relatively stable to minimize flakes, so when tests are + # grouped (eg. batched tests), we cannot perfectly fill all paritions as that + # would require breaking up groups. + def _PartitionTests(self, tests, num_desired_partitions, max_partition_size): + # pylint: disable=no-self-use + partitions = [] + + # Sort by hash so we don't put all tests in a slow suite in the same + # partition. + tests = sorted( + tests, + key=lambda t: hash( + self._GetUniqueTestName(t[0] if isinstance(t, list) else t))) + + def CountTestsIndividually(test): + if not isinstance(test, list): + return False + annotations = test[0]['annotations'] + # UnitTests tests are really fast, so to balance shards better, count + # UnitTests Batches as single tests. + return ('Batch' not in annotations + or annotations['Batch']['value'] != 'UnitTests') + + num_not_yet_allocated = sum( + [len(test) - 1 for test in tests if CountTestsIndividually(test)]) + num_not_yet_allocated += len(tests) + + # Fast linear partition approximation capped by max_partition_size. We + # cannot round-robin or otherwise re-order tests dynamically because we want + # test order to remain stable. + partition_size = min(num_not_yet_allocated // num_desired_partitions, + max_partition_size) + partitions.append([]) + last_partition_size = 0 + for test in tests: + test_count = len(test) if CountTestsIndividually(test) else 1 + num_not_yet_allocated -= test_count + # Make a new shard whenever we would overfill the previous one. However, + # if the size of the test group is larger than the max partition size on + # its own, just put the group in its own shard instead of splitting up the + # group. + if (last_partition_size + test_count > partition_size + and last_partition_size > 0): + num_desired_partitions -= 1 + partitions.append([]) + partitions[-1].append(test) + last_partition_size = test_count + if num_desired_partitions <= 0: + # Too many tests for number of partitions, just fill all partitions + # beyond num_desired_partitions. + partition_size = max_partition_size + else: + # Re-balance remaining partitions. + partition_size = min(num_not_yet_allocated // num_desired_partitions, + max_partition_size) + else: + partitions[-1].append(test) + last_partition_size += test_count + + if not partitions[-1]: + partitions.pop() + return partitions + def GetTool(self, device): if str(device) not in self._tools: self._tools[str(device)] = valgrind_tools.CreateTool( diff --git a/chromium/build/android/pylib/local/emulator/avd.py b/chromium/build/android/pylib/local/emulator/avd.py index 452887ca207..358700d6870 100644 --- a/chromium/build/android/pylib/local/emulator/avd.py +++ b/chromium/build/android/pylib/local/emulator/avd.py @@ -375,7 +375,7 @@ class AvdConfig(object): pkgs_by_dir[pkg.dest_path] = [] pkgs_by_dir[pkg.dest_path].append(pkg) - for pkg_dir, pkgs in pkgs_by_dir.iteritems(): + for pkg_dir, pkgs in pkgs_by_dir.items(): logging.info('Installing packages in %s', pkg_dir) cipd_root = os.path.join(constants.DIR_SOURCE_ROOT, pkg_dir) if not os.path.exists(cipd_root): @@ -530,6 +530,10 @@ class _AvdInstance(object): '-report-console', 'unix:%s' % socket_path, '-no-boot-anim', + # Set the gpu mode to swiftshader_indirect otherwise the avd may exit + # with the error "change of render" under window mode + '-gpu', + 'swiftshader_indirect', ] if read_only: diff --git a/chromium/build/android/pylib/local/emulator/ini.py b/chromium/build/android/pylib/local/emulator/ini.py index 45761884fcb..fe363c99274 100644 --- a/chromium/build/android/pylib/local/emulator/ini.py +++ b/chromium/build/android/pylib/local/emulator/ini.py @@ -27,7 +27,7 @@ def load(fp): def dumps(obj): ret = '' - for k, v in sorted(obj.iteritems()): + for k, v in sorted(obj.items()): ret += '%s = %s\n' % (k, str(v)) return ret diff --git a/chromium/build/android/pylib/local/emulator/ini_test.py b/chromium/build/android/pylib/local/emulator/ini_test.py index 4bc9ddb9ef5..22f884d41fe 100755 --- a/chromium/build/android/pylib/local/emulator/ini_test.py +++ b/chromium/build/android/pylib/local/emulator/ini_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env vpython # Copyright 2020 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/pylib/output/local_output_manager_test.py b/chromium/build/android/pylib/output/local_output_manager_test.py index 12452a6616a..7954350c032 100755 --- a/chromium/build/android/pylib/output/local_output_manager_test.py +++ b/chromium/build/android/pylib/output/local_output_manager_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env vpython # Copyright 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/pylib/output/noop_output_manager_test.py b/chromium/build/android/pylib/output/noop_output_manager_test.py index c735a0469af..4e470efc0ed 100755 --- a/chromium/build/android/pylib/output/noop_output_manager_test.py +++ b/chromium/build/android/pylib/output/noop_output_manager_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env vpython # Copyright 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/pylib/output/remote_output_manager_test.py b/chromium/build/android/pylib/output/remote_output_manager_test.py index d87c6eb3a9c..4c6c081003e 100755 --- a/chromium/build/android/pylib/output/remote_output_manager_test.py +++ b/chromium/build/android/pylib/output/remote_output_manager_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env vpython # Copyright 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/pylib/results/json_results_test.py b/chromium/build/android/pylib/results/json_results_test.py index f410adef696..66473311853 100755 --- a/chromium/build/android/pylib/results/json_results_test.py +++ b/chromium/build/android/pylib/results/json_results_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/pylib/utils/decorators_test.py b/chromium/build/android/pylib/utils/decorators_test.py index 60f4811b4f4..73a9f0de669 100755 --- a/chromium/build/android/pylib/utils/decorators_test.py +++ b/chromium/build/android/pylib/utils/decorators_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/pylib/utils/device_dependencies.py b/chromium/build/android/pylib/utils/device_dependencies.py index ff78f9f04e6..9cb5bd892a8 100644 --- a/chromium/build/android/pylib/utils/device_dependencies.py +++ b/chromium/build/android/pylib/utils/device_dependencies.py @@ -65,7 +65,7 @@ def DevicePathComponentsFor(host_path, output_directory): e.g., given - '$CHROMIUM_SRC/foo/bar/baz.txt' + '$RUNTIME_DEPS_ROOT_DIR/foo/bar/baz.txt' this would return @@ -73,12 +73,16 @@ def DevicePathComponentsFor(host_path, output_directory): This handles a couple classes of paths differently than it otherwise would: - All .pak files get mapped to top-level paks/ - - Anything in the output directory gets mapped relative to the output - directory rather than the source directory. + - All other dependencies get mapped to the top level directory + - If a file is not in the output directory then it's relative path to + the output directory will start with .. strings, so we remove those + and then the path gets mapped to the top-level directory + - If a file is in the output directory then the relative path to the + output directory gets mapped to the top-level directory e.g. given - '$CHROMIUM_SRC/out/Release/icu_fake_dir/icudtl.dat' + '$RUNTIME_DEPS_ROOT_DIR/out/Release/icu_fake_dir/icudtl.dat' this would return @@ -89,18 +93,20 @@ def DevicePathComponentsFor(host_path, output_directory): Returns: A list of device path components. """ - if host_path.startswith(output_directory): - if os.path.splitext(host_path)[1] == '.pak': - return [None, 'paks', os.path.basename(host_path)] - rel_host_path = os.path.relpath(host_path, output_directory) - else: - rel_host_path = os.path.relpath(host_path, constants.DIR_SOURCE_ROOT) + if (host_path.startswith(output_directory) and + os.path.splitext(host_path)[1] == '.pak'): + return [None, 'paks', os.path.basename(host_path)] + + rel_host_path = os.path.relpath(host_path, output_directory) device_path_components = [None] p = rel_host_path while p: p, d = os.path.split(p) - if d: + # The relative path from the output directory to a file under the runtime + # deps root directory may start with multiple .. strings, so they need to + # be skipped. + if d and d != os.pardir: device_path_components.insert(1, d) return device_path_components diff --git a/chromium/build/android/pylib/utils/device_dependencies_test.py b/chromium/build/android/pylib/utils/device_dependencies_test.py index aaa9ebf68a3..b2da5a7ee58 100755 --- a/chromium/build/android/pylib/utils/device_dependencies_test.py +++ b/chromium/build/android/pylib/utils/device_dependencies_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env vpython # Copyright 2016 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/pylib/utils/dexdump_test.py b/chromium/build/android/pylib/utils/dexdump_test.py index 6b2c4542f2b..3197853fc00 100755 --- a/chromium/build/android/pylib/utils/dexdump_test.py +++ b/chromium/build/android/pylib/utils/dexdump_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env vpython # Copyright 2016 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/pylib/utils/maven_downloader.py b/chromium/build/android/pylib/utils/maven_downloader.py index c60b0140acc..1dc1542ea19 100755 --- a/chromium/build/android/pylib/utils/maven_downloader.py +++ b/chromium/build/android/pylib/utils/maven_downloader.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,7 +7,10 @@ import errno import logging import os import shutil +import sys +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) +import devil_chromium # pylint: disable=unused-import from devil.utils import cmd_helper from devil.utils import parallelizer diff --git a/chromium/build/android/pylib/utils/proguard_test.py b/chromium/build/android/pylib/utils/proguard_test.py index 7672476e0a6..b11c299580e 100755 --- a/chromium/build/android/pylib/utils/proguard_test.py +++ b/chromium/build/android/pylib/utils/proguard_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env vpython # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/resource_sizes.gni b/chromium/build/android/resource_sizes.gni index 5bc430ac1ef..2c91749c5ee 100644 --- a/chromium/build/android/resource_sizes.gni +++ b/chromium/build/android/resource_sizes.gni @@ -32,7 +32,10 @@ template("android_resource_sizes_test") { "@WrappedPath(.)", ] - data = [] + data = [ + "//.vpython", + "//.vpython3", + ] if (defined(invoker.trichrome_chrome_path)) { data += [ invoker.trichrome_chrome_path, diff --git a/chromium/build/android/resource_sizes.py b/chromium/build/android/resource_sizes.py index dd51525ae67..c59297072a8 100755 --- a/chromium/build/android/resource_sizes.py +++ b/chromium/build/android/resource_sizes.py @@ -159,7 +159,7 @@ def _MeasureApkSignatureBlock(zip_file): start_of_central_directory = struct.unpack('<I', zip_file.fp.read(4))[0] # Compute the offset after the last zip entry. - last_info = zip_file.infolist()[-1] + last_info = max(zip_file.infolist(), key=lambda i: i.header_offset) last_header_size = (30 + len(last_info.filename) + _ReadZipInfoExtraFieldLength(zip_file, last_info)) end_of_last_file = (last_info.header_offset + last_header_size + diff --git a/chromium/build/android/resource_sizes.pydeps b/chromium/build/android/resource_sizes.pydeps index 8be61c41f9c..d956f5bae72 100644 --- a/chromium/build/android/resource_sizes.pydeps +++ b/chromium/build/android/resource_sizes.pydeps @@ -35,9 +35,9 @@ ../../third_party/catapult/devil/devil/utils/reraiser_thread.py ../../third_party/catapult/devil/devil/utils/timeout_retry.py ../../third_party/catapult/devil/devil/utils/watchdog_timer.py +../../third_party/catapult/third_party/six/six.py ../../third_party/catapult/third_party/vinn/vinn/__init__.py ../../third_party/catapult/third_party/vinn/vinn/_vinn.py -../../third_party/catapult/third_party/zipfile/zipfile_2_7_13.py ../../third_party/catapult/tracing/tracing/__init__.py ../../third_party/catapult/tracing/tracing/value/__init__.py ../../third_party/catapult/tracing/tracing/value/convert_chart_json.py diff --git a/chromium/build/android/screenshot.py b/chromium/build/android/screenshot.py index 6ab906086d7..523d859a215 100755 --- a/chromium/build/android/screenshot.py +++ b/chromium/build/android/screenshot.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/stacktrace/crashpad_stackwalker.py b/chromium/build/android/stacktrace/crashpad_stackwalker.py index beeb6f4b14d..9616a54ba65 100755 --- a/chromium/build/android/stacktrace/crashpad_stackwalker.py +++ b/chromium/build/android/stacktrace/crashpad_stackwalker.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # # Copyright 2019 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/chromium/build/android/stacktrace/java_deobfuscate_test.py b/chromium/build/android/stacktrace/java_deobfuscate_test.py index 98b66dd02e0..1bf81c959c5 100755 --- a/chromium/build/android/stacktrace/java_deobfuscate_test.py +++ b/chromium/build/android/stacktrace/java_deobfuscate_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # # Copyright 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/chromium/build/android/stacktrace/stackwalker.py b/chromium/build/android/stacktrace/stackwalker.py index 5fbab33920a..4f2782fc693 100755 --- a/chromium/build/android/stacktrace/stackwalker.py +++ b/chromium/build/android/stacktrace/stackwalker.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # # Copyright 2016 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/chromium/build/android/test/nocompile_gn/BUILD.gn b/chromium/build/android/test/nocompile_gn/BUILD.gn index 3334b6d02c0..d3262feeba2 100644 --- a/chromium/build/android/test/nocompile_gn/BUILD.gn +++ b/chromium/build/android/test/nocompile_gn/BUILD.gn @@ -32,6 +32,9 @@ template("lint_test") { build_config_dep = "$_apk_target$build_config_target_suffix" build_config = get_label_info(_apk_target, "target_gen_dir") + "/" + get_label_info(_apk_target, "name") + ".build_config" + if (enable_android_nocompile_tests) { + skip_build_server = true + } } } diff --git a/chromium/build/android/test_runner.py b/chromium/build/android/test_runner.py index 07c46002d4f..ae7d1bcb423 100755 --- a/chromium/build/android/test_runner.py +++ b/chromium/build/android/test_runner.py @@ -904,6 +904,27 @@ def RunTestsInPlatformMode(args, result_sink_client=None): global_tags=list(global_results_tags), indent=2) + test_class_to_file_name_dict = {} + # Test Location is only supported for instrumentation tests as it + # requires the size-info file. + if test_instance.TestType() == 'instrumentation': + test_class_to_file_name_dict = _CreateClassToFileNameDict(args.test_apk) + + if result_sink_client: + for run in all_raw_results: + for results in run: + for r in results.GetAll(): + # Matches chrome.page_info.PageInfoViewTest#testChromePage + match = re.search(r'^(.+\..+)#', r.GetName()) + test_file_name = test_class_to_file_name_dict.get( + match.group(1)) if match else None + # Some tests put in non utf-8 char as part of the test + # which breaks uploads, so need to decode and re-encode. + result_sink_client.Post( + r.GetName(), r.GetType(), r.GetDuration(), + r.GetLog().decode('utf-8', 'replace').encode('utf-8'), + test_file_name) + @contextlib.contextmanager def upload_logcats_file(): try: @@ -964,29 +985,11 @@ def RunTestsInPlatformMode(args, result_sink_client=None): for r in reversed(raw_results): iteration_results.AddTestRunResults(r) all_iteration_results.append(iteration_results) - - test_class_to_file_name_dict = {} - # Test Location is only supported for instrumentation tests as it - # requires the size-info file. - if test_instance.TestType() == 'instrumentation': - test_class_to_file_name_dict = _CreateClassToFileNameDict( - args.test_apk) - iteration_count += 1 - for r in iteration_results.GetAll(): - if result_sink_client: - # Matches chrome.page_info.PageInfoViewTest#testChromePage - match = re.search(r'^(.+\..+)#', r.GetName()) - test_file_name = test_class_to_file_name_dict.get( - match.group(1)) if match else None - # Some tests put in non utf-8 char as part of the test - # which breaks uploads, so need to decode and re-encode. - result_sink_client.Post( - r.GetName(), r.GetType(), - r.GetLog().decode('utf-8', 'replace').encode('utf-8'), - test_file_name) + for r in iteration_results.GetAll(): result_counts[r.GetName()][r.GetType()] += 1 + report_results.LogFull( results=iteration_results, test_type=test_instance.TestType(), diff --git a/chromium/build/android/test_runner.pydeps b/chromium/build/android/test_runner.pydeps index a8dd5bf0886..63c30d83552 100644 --- a/chromium/build/android/test_runner.pydeps +++ b/chromium/build/android/test_runner.pydeps @@ -95,7 +95,7 @@ ../../third_party/catapult/devil/devil/utils/timeout_retry.py ../../third_party/catapult/devil/devil/utils/watchdog_timer.py ../../third_party/catapult/devil/devil/utils/zip_utils.py -../../third_party/catapult/third_party/zipfile/zipfile_2_7_13.py +../../third_party/catapult/third_party/six/six.py ../../third_party/colorama/src/colorama/__init__.py ../../third_party/colorama/src/colorama/ansi.py ../../third_party/colorama/src/colorama/ansitowin32.py diff --git a/chromium/build/android/test_wrapper/logdog_wrapper.py b/chromium/build/android/test_wrapper/logdog_wrapper.py index fdb3fd3dab2..782d5d87abf 100755 --- a/chromium/build/android/test_wrapper/logdog_wrapper.py +++ b/chromium/build/android/test_wrapper/logdog_wrapper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2016 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/build/android/tombstones.py b/chromium/build/android/tombstones.py index 33fd6814468..082e7c1c783 100755 --- a/chromium/build/android/tombstones.py +++ b/chromium/build/android/tombstones.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/chromium/build/android/update_verification.py b/chromium/build/android/update_verification.py index a6529cdb390..3d478f4cf00 100755 --- a/chromium/build/android/update_verification.py +++ b/chromium/build/android/update_verification.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/chromium/build/android/video_recorder.py b/chromium/build/android/video_recorder.py index b21759a35a9..6c54e7a55f7 100755 --- a/chromium/build/android/video_recorder.py +++ b/chromium/build/android/video_recorder.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. |