diff options
Diffstat (limited to 'chromium/build/android/pylib/local')
8 files changed, 908 insertions, 131 deletions
diff --git a/chromium/build/android/pylib/local/device/local_device_gtest_run.py b/chromium/build/android/pylib/local/device/local_device_gtest_run.py index 5044cdf1247..605b826c1a0 100644 --- a/chromium/build/android/pylib/local/device/local_device_gtest_run.py +++ b/chromium/build/android/pylib/local/device/local_device_gtest_run.py @@ -10,11 +10,13 @@ import posixpath import shutil import time +from devil import base_error from devil.android import crash_handler from devil.android import device_errors from devil.android import device_temp_file from devil.android import logcat_monitor from devil.android import ports +from devil.android.sdk import version_codes from devil.utils import reraiser_thread from incremental_install import installer from pylib import constants @@ -35,6 +37,8 @@ _EXTRA_COMMAND_LINE_FILE = ( 'org.chromium.native_test.NativeTest.CommandLineFile') _EXTRA_COMMAND_LINE_FLAGS = ( 'org.chromium.native_test.NativeTest.CommandLineFlags') +_EXTRA_COVERAGE_DEVICE_FILE = ( + 'org.chromium.native_test.NativeTest.CoverageDeviceFile') _EXTRA_STDOUT_FILE = ( 'org.chromium.native_test.NativeTestInstrumentationTestRunner' '.StdoutFile') @@ -102,6 +106,24 @@ def _ExtractTestsFromFilter(gtest_filter): return patterns +def _PullCoverageFile(device, coverage_device_file, output_dir): + """Pulls coverage file on device to host directory. + + Args: + device: The working device. + coverage_device_file: The temporary coverage file on device. + output_dir: The output directory on host. + """ + try: + if not os.path.exists(output_dir): + os.makedirs(output_dir) + device.PullFile(coverage_device_file.name, output_dir) + except (OSError, base_error.BaseError) as e: + logging.warning('Failed to handle coverage data after tests: %s', e) + finally: + coverage_device_file.close() + + class _ApkDelegate(object): def __init__(self, test_instance, tool): self._activity = test_instance.activity @@ -116,6 +138,7 @@ class _ApkDelegate(object): self._extras = test_instance.extras self._wait_for_java_debugger = test_instance.wait_for_java_debugger self._tool = tool + self._coverage_dir = test_instance.coverage_dir def GetTestDataRoot(self, device): # pylint: disable=no-self-use @@ -138,6 +161,15 @@ class _ApkDelegate(object): def Run(self, test, device, flags=None, **kwargs): extras = dict(self._extras) + device_api = device.build_version_sdk + + if self._coverage_dir and device_api >= version_codes.LOLLIPOP: + coverage_device_file = device_temp_file.DeviceTempFile( + device.adb, + suffix='.profraw', + prefix=self._suite, + dir=device.GetExternalStoragePath()) + extras[_EXTRA_COVERAGE_DEVICE_FILE] = coverage_device_file.name if ('timeout' in kwargs and gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT not in extras): @@ -193,6 +225,10 @@ class _ApkDelegate(object): except Exception: device.ForceStop(self._package) raise + finally: + if self._coverage_dir and device_api >= version_codes.LOLLIPOP: + _PullCoverageFile(device, coverage_device_file, self._coverage_dir) + # TODO(jbudorick): Remove this after resolving crbug.com/726880 if device.PathExists(stdout_file.name): logging.info('%s size on device: %s', stdout_file.name, @@ -218,13 +254,18 @@ class _ApkDelegate(object): class _ExeDelegate(object): - def __init__(self, tr, dist_dir, tool): - self._host_dist_dir = dist_dir - self._exe_file_name = os.path.basename(dist_dir)[:-len('__dist')] + + def __init__(self, tr, test_instance, tool): + self._host_dist_dir = test_instance.exe_dist_dir + self._exe_file_name = os.path.basename( + test_instance.exe_dist_dir)[:-len('__dist')] self._device_dist_dir = posixpath.join( - constants.TEST_EXECUTABLE_DIR, os.path.basename(dist_dir)) + constants.TEST_EXECUTABLE_DIR, + os.path.basename(test_instance.exe_dist_dir)) self._test_run = tr self._tool = tool + self._coverage_dir = test_instance.coverage_dir + self._suite = test_instance.suite def GetTestDataRoot(self, device): # pylint: disable=no-self-use @@ -261,6 +302,14 @@ class _ExeDelegate(object): 'LD_LIBRARY_PATH': self._device_dist_dir } + if self._coverage_dir: + coverage_device_file = device_temp_file.DeviceTempFile( + device.adb, + suffix='.profraw', + prefix=self._suite, + dir=device.GetExternalStoragePath()) + env['LLVM_PROFILE_FILE'] = coverage_device_file.name + if self._tool != 'asan': env['UBSAN_OPTIONS'] = constants.UBSAN_OPTIONS @@ -276,6 +325,10 @@ class _ExeDelegate(object): # fine from the test runner's perspective; thus check_return=False. output = device.RunShellCommand( cmd, cwd=cwd, env=env, check_return=False, large_output=True, **kwargs) + + if self._coverage_dir: + _PullCoverageFile(device, coverage_device_file, self._coverage_dir) + return output def PullAppFiles(self, device, files, directory): @@ -296,8 +349,7 @@ class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun): if self._test_instance.apk: self._delegate = _ApkDelegate(self._test_instance, env.tool) elif self._test_instance.exe_dist_dir: - self._delegate = _ExeDelegate(self, self._test_instance.exe_dist_dir, - self._env.tool) + self._delegate = _ExeDelegate(self, self._test_instance, self._env.tool) if self._test_instance.isolated_script_test_perf_output: self._test_perf_output_filenames = _GenerateSequentialFileNames( self._test_instance.isolated_script_test_perf_output) diff --git a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py index 18914e9aa13..25e47a0159d 100644 --- a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py +++ b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py @@ -190,32 +190,33 @@ class LocalDeviceInstrumentationTestRun( steps.append(use_webview_provider) - def install_helper(apk, permissions): + def install_helper(apk, modules=None, fake_modules=None, + permissions=None): + @instrumentation_tracing.no_tracing - @trace_event.traced("apk_path") - def install_helper_internal(d, apk_path=apk.path): + @trace_event.traced + def install_helper_internal(d, apk_path=None): # pylint: disable=unused-argument - d.Install(apk, permissions=permissions) + logging.info('Start Installing %s', apk.path) + d.Install( + apk, + modules=modules, + fake_modules=fake_modules, + permissions=permissions) + logging.info('Finished Installing %s', apk.path) + return install_helper_internal def incremental_install_helper(apk, json_path, permissions): - @trace_event.traced("apk_path") - def incremental_install_helper_internal(d, apk_path=apk.path): + + @trace_event.traced + def incremental_install_helper_internal(d, apk_path=None): # pylint: disable=unused-argument + logging.info('Start Incremental Installing %s', apk.path) installer.Install(d, json_path, apk=apk, permissions=permissions) - return incremental_install_helper_internal + logging.info('Finished Incremental Installing %s', apk.path) - if self._test_instance.apk_under_test: - permissions = self._test_instance.apk_under_test.GetPermissions() - if self._test_instance.apk_under_test_incremental_install_json: - steps.append(incremental_install_helper( - self._test_instance.apk_under_test, - self._test_instance. - apk_under_test_incremental_install_json, - permissions)) - else: - steps.append(install_helper(self._test_instance.apk_under_test, - permissions)) + return incremental_install_helper_internal permissions = self._test_instance.test_apk.GetPermissions() if self._test_instance.test_apk_incremental_install_json: @@ -225,11 +226,29 @@ class LocalDeviceInstrumentationTestRun( test_apk_incremental_install_json, permissions)) else: - steps.append(install_helper(self._test_instance.test_apk, - permissions)) + steps.append( + install_helper( + self._test_instance.test_apk, permissions=permissions)) + + steps.extend( + install_helper(apk) for apk in self._test_instance.additional_apks) - steps.extend(install_helper(apk, None) - for apk in self._test_instance.additional_apks) + # The apk under test needs to be installed last since installing other + # apks after will unintentionally clear the fake module directory. + # TODO(wnwen): Make this more robust, fix crbug.com/1010954. + if self._test_instance.apk_under_test: + permissions = self._test_instance.apk_under_test.GetPermissions() + if self._test_instance.apk_under_test_incremental_install_json: + steps.append( + incremental_install_helper( + self._test_instance.apk_under_test, + self._test_instance.apk_under_test_incremental_install_json, + permissions)) + else: + steps.append( + install_helper(self._test_instance.apk_under_test, + self._test_instance.modules, + self._test_instance.fake_modules, permissions)) @trace_event.traced def set_debug_app(dev): @@ -282,9 +301,9 @@ class LocalDeviceInstrumentationTestRun( host_device_tuples_substituted = [ (h, local_device_test_run.SubstituteDeviceRoot(d, device_root)) for h, d in host_device_tuples] - logging.info('instrumentation data deps:') + logging.info('Pushing data dependencies.') for h, d in host_device_tuples_substituted: - logging.info('%r -> %r', h, d) + logging.debug(' %r -> %r', h, d) dev.PushChangedFiles(host_device_tuples_substituted, delete_device_stale=True) if not host_device_tuples_substituted: @@ -541,6 +560,7 @@ class LocalDeviceInstrumentationTestRun( with ui_capture_dir: with self._env.output_manager.ArchivedTempfile( stream_name, 'logcat') as logcat_file: + logmon = None try: with logcat_monitor.LogcatMonitor( device.adb, @@ -555,7 +575,8 @@ class LocalDeviceInstrumentationTestRun( output = device.StartInstrumentation( target, raw=True, extras=extras, timeout=timeout, retries=0) finally: - logmon.Close() + if logmon: + logmon.Close() if logcat_file.Link(): logging.info('Logcat saved to %s', logcat_file.Link()) @@ -589,13 +610,12 @@ class LocalDeviceInstrumentationTestRun( def handle_coverage_data(): if self._test_instance.coverage_directory: try: + if not os.path.exists(self._test_instance.coverage_directory): + os.makedirs(self._test_instance.coverage_directory) device.PullFile(coverage_device_file, self._test_instance.coverage_directory) - device.RunShellCommand( - 'rm -f %s' % posixpath.join(coverage_directory, '*'), - check_return=True, - shell=True) - except base_error.BaseError as e: + device.RemovePath(coverage_device_file, True) + except (OSError, base_error.BaseError) as e: logging.warning('Failed to handle coverage data after tests: %s', e) def handle_render_test_data(): diff --git a/chromium/build/android/pylib/local/emulator/avd.py b/chromium/build/android/pylib/local/emulator/avd.py new file mode 100644 index 00000000000..fab9061e90e --- /dev/null +++ b/chromium/build/android/pylib/local/emulator/avd.py @@ -0,0 +1,496 @@ +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import contextlib +import json +import logging +import os +import socket +import stat +import subprocess +import textwrap +import threading + +from google.protobuf import text_format # pylint: disable=import-error + +from devil.android import device_utils +from devil.android.sdk import adb_wrapper +from devil.utils import cmd_helper +from devil.utils import timeout_retry +from py_utils import tempfile_ext +from pylib import constants +from pylib.local.emulator.proto import avd_pb2 + +_ALL_PACKAGES = object() +_DEFAULT_AVDMANAGER_PATH = os.path.join(constants.ANDROID_SDK_ROOT, 'tools', + 'bin', 'avdmanager') + + +class AvdException(Exception): + """Raised when this module has a problem interacting with an AVD.""" + + def __init__(self, summary, command=None, stdout=None, stderr=None): + message_parts = [summary] + if command: + message_parts.append(' command: %s' % ' '.join(command)) + if stdout: + message_parts.append(' stdout:') + message_parts.extend(' %s' % line for line in stdout.splitlines()) + if stderr: + message_parts.append(' stderr:') + message_parts.extend(' %s' % line for line in stderr.splitlines()) + + super(AvdException, self).__init__('\n'.join(message_parts)) + + +def _Load(avd_proto_path): + """Loads an Avd proto from a textpb file at the given path. + + Should not be called outside of this module. + + Args: + avd_proto_path: path to a textpb file containing an Avd message. + """ + with open(avd_proto_path) as avd_proto_file: + return text_format.Merge(avd_proto_file.read(), avd_pb2.Avd()) + + +class _AvdManagerAgent(object): + """Private utility for interacting with avdmanager.""" + + def __init__(self, avd_home, sdk_root): + """Create an _AvdManagerAgent. + + Args: + avd_home: path to ANDROID_AVD_HOME directory. + Typically something like /path/to/dir/.android/avd + sdk_root: path to SDK root directory. + """ + self._avd_home = avd_home + self._sdk_root = sdk_root + + self._env = dict(os.environ) + + # avdmanager, like many tools that have evolved from `android` + # (http://bit.ly/2m9JiTx), uses toolsdir to find the SDK root. + # Pass avdmanager a fake directory under the directory in which + # we install the system images s.t. avdmanager can find the + # system images. + fake_tools_dir = os.path.join(self._sdk_root, 'non-existent-tools') + self._env.update({ + 'ANDROID_AVD_HOME': + self._avd_home, + 'AVDMANAGER_OPTS': + '-Dcom.android.sdkmanager.toolsdir=%s' % fake_tools_dir, + }) + + def Create(self, avd_name, system_image, force=False): + """Call `avdmanager create`. + + Args: + avd_name: name of the AVD to create. + system_image: system image to use for the AVD. + force: whether to force creation, overwriting any existing + AVD with the same name. + """ + create_cmd = [ + _DEFAULT_AVDMANAGER_PATH, + '-v', + 'create', + 'avd', + '-n', + avd_name, + '-k', + system_image, + ] + if force: + create_cmd += ['--force'] + + create_proc = cmd_helper.Popen( + create_cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self._env) + output, error = create_proc.communicate(input='\n') + if create_proc.returncode != 0: + raise AvdException( + 'AVD creation failed', + command=create_cmd, + stdout=output, + stderr=error) + + for line in output.splitlines(): + logging.info(' %s', line) + + def Delete(self, avd_name): + """Call `avdmanager delete`. + + Args: + avd_name: name of the AVD to delete. + """ + delete_cmd = [ + _DEFAULT_AVDMANAGER_PATH, + '-v', + 'delete', + 'avd', + '-n', + avd_name, + ] + try: + for line in cmd_helper.IterCmdOutputLines(delete_cmd, env=self._env): + logging.info(' %s', line) + except subprocess.CalledProcessError as e: + raise AvdException('AVD deletion failed: %s' % str(e), command=delete_cmd) + + +class AvdConfig(object): + """Represents a particular AVD configuration. + + This class supports creation, installation, and execution of an AVD + from a given Avd proto message, as defined in + //build/android/pylib/local/emulator/proto/avd.proto. + """ + + def __init__(self, avd_proto_path): + """Create an AvdConfig object. + + Args: + avd_proto_path: path to a textpb file containing an Avd message. + """ + self._config = _Load(avd_proto_path) + + self._emulator_home = os.path.join(constants.DIR_SOURCE_ROOT, + self._config.avd_package.dest_path) + self._emulator_sdk_root = os.path.join( + constants.DIR_SOURCE_ROOT, self._config.emulator_package.dest_path) + self._emulator_path = os.path.join(self._emulator_sdk_root, 'emulator', + 'emulator') + + self._initialized = False + self._initializer_lock = threading.Lock() + + def Create(self, + force=False, + snapshot=False, + keep=False, + cipd_json_output=None): + """Create an instance of the AVD CIPD package. + + This method: + - installs the requisite system image + - creates the AVD + - modifies the AVD's ini files to support running chromium tests + in chromium infrastructure + - optionally starts & stops the AVD for snapshotting (default no) + - creates and uploads an instance of the AVD CIPD package + - optionally deletes the AVD (default yes) + + Args: + force: bool indicating whether to force create the AVD. + snapshot: bool indicating whether to snapshot the AVD before creating + the CIPD package. + keep: bool indicating whether to keep the AVD after creating + the CIPD package. + cipd_json_output: string path to pass to `cipd create` via -json-output. + """ + logging.info('Installing required packages.') + self.Install(packages=[ + self._config.emulator_package, + self._config.system_image_package, + ]) + + android_avd_home = os.path.join(self._emulator_home, 'avd') + + if not os.path.exists(android_avd_home): + os.makedirs(android_avd_home) + + avd_manager = _AvdManagerAgent( + avd_home=android_avd_home, sdk_root=self._emulator_sdk_root) + + logging.info('Creating AVD.') + avd_manager.Create( + avd_name=self._config.avd_name, + system_image=self._config.system_image_name, + force=force) + + try: + logging.info('Modifying AVD configuration.') + + # Clear out any previous configuration or state from this AVD. + root_ini = os.path.join(android_avd_home, + '%s.ini' % self._config.avd_name) + avd_dir = os.path.join(android_avd_home, '%s.avd' % self._config.avd_name) + config_ini = os.path.join(avd_dir, 'config.ini') + + with open(root_ini, 'a') as root_ini_file: + root_ini_file.write('path.rel=avd/%s.avd\n' % self._config.avd_name) + + with open(config_ini, 'a') as config_ini_file: + config_ini_file.write( + textwrap.dedent("""\ + disk.dataPartition.size=4G + hw.lcd.density=160 + hw.lcd.height=960 + hw.lcd.width=480 + """)) + + # Start & stop the AVD. + self._Initialize() + instance = _AvdInstance(self._emulator_path, self._config.avd_name, + self._emulator_home) + instance.Start(read_only=False, snapshot_save=snapshot) + device_utils.DeviceUtils(instance.serial).WaitUntilFullyBooted( + timeout=180, retries=0) + instance.Stop() + + # The multiinstance lock file seems to interfere with the emulator's + # operation in some circumstances (beyond the obvious -read-only ones), + # and there seems to be no mechanism by which it gets closed or deleted. + # See https://bit.ly/2pWQTH7 for context. + multiInstanceLockFile = os.path.join(avd_dir, 'multiinstance.lock') + if os.path.exists(multiInstanceLockFile): + os.unlink(multiInstanceLockFile) + + package_def_content = { + 'package': + self._config.avd_package.package_name, + 'root': + self._emulator_home, + 'install_mode': + 'copy', + 'data': [ + { + 'dir': os.path.relpath(avd_dir, self._emulator_home) + }, + { + 'file': os.path.relpath(root_ini, self._emulator_home) + }, + ], + } + + logging.info('Creating AVD CIPD package.') + logging.debug('ensure file content: %s', + json.dumps(package_def_content, indent=2)) + + with tempfile_ext.TemporaryFileName(suffix='.json') as package_def_path: + with open(package_def_path, 'w') as package_def_file: + json.dump(package_def_content, package_def_file) + + logging.info(' %s', self._config.avd_package.package_name) + cipd_create_cmd = [ + 'cipd', + 'create', + '-pkg-def', + package_def_path, + ] + if cipd_json_output: + cipd_create_cmd.extend([ + '-json-output', + cipd_json_output, + ]) + try: + for line in cmd_helper.IterCmdOutputLines(cipd_create_cmd): + logging.info(' %s', line) + except subprocess.CalledProcessError as e: + raise AvdException( + 'CIPD package creation failed: %s' % str(e), + command=cipd_create_cmd) + + finally: + if not keep: + logging.info('Deleting AVD.') + avd_manager.Delete(avd_name=self._config.avd_name) + + def Install(self, packages=_ALL_PACKAGES): + """Installs the requested CIPD packages. + + Returns: None + Raises: AvdException on failure to install. + """ + pkgs_by_dir = {} + if packages is _ALL_PACKAGES: + packages = [ + self._config.avd_package, + self._config.emulator_package, + self._config.system_image_package, + ] + for pkg in packages: + if not pkg.dest_path in pkgs_by_dir: + pkgs_by_dir[pkg.dest_path] = [] + pkgs_by_dir[pkg.dest_path].append(pkg) + + for pkg_dir, pkgs in pkgs_by_dir.iteritems(): + logging.info('Installing packages in %s', pkg_dir) + cipd_root = os.path.join(constants.DIR_SOURCE_ROOT, pkg_dir) + if not os.path.exists(cipd_root): + os.makedirs(cipd_root) + ensure_path = os.path.join(cipd_root, '.ensure') + with open(ensure_path, 'w') as ensure_file: + # Make CIPD ensure that all files are present, even if + # it thinks the package is installed. + ensure_file.write('$ParanoidMode CheckPresence\n\n') + for pkg in pkgs: + ensure_file.write('%s %s\n' % (pkg.package_name, pkg.version)) + logging.info(' %s %s', pkg.package_name, pkg.version) + ensure_cmd = [ + 'cipd', + 'ensure', + '-ensure-file', + ensure_path, + '-root', + cipd_root, + ] + try: + for line in cmd_helper.IterCmdOutputLines(ensure_cmd): + logging.info(' %s', line) + except subprocess.CalledProcessError as e: + raise AvdException( + 'Failed to install CIPD package %s: %s' % (pkg.package_name, + str(e)), + command=ensure_cmd) + + # The emulator requires that some files are writable. + for dirname, _, filenames in os.walk(self._emulator_home): + for f in filenames: + path = os.path.join(dirname, f) + mode = os.lstat(path).st_mode + if mode & stat.S_IRUSR: + mode = mode | stat.S_IWUSR + os.chmod(path, mode) + + def _Initialize(self): + if self._initialized: + return + + with self._initializer_lock: + if self._initialized: + return + + # Emulator start-up looks for the adb daemon. Make sure it's running. + adb_wrapper.AdbWrapper.StartServer() + + # Emulator start-up tries to check for the SDK root by looking for + # platforms/ and platform-tools/. Ensure they exist. + # See http://bit.ly/2YAkyFE for context. + required_dirs = [ + os.path.join(self._emulator_sdk_root, 'platforms'), + os.path.join(self._emulator_sdk_root, 'platform-tools'), + ] + for d in required_dirs: + if not os.path.exists(d): + os.makedirs(d) + + def CreateInstance(self): + """Creates an AVD instance without starting it. + + Returns: + An _AvdInstance. + """ + self._Initialize() + return _AvdInstance(self._emulator_path, self._config.avd_name, + self._emulator_home) + + def StartInstance(self): + """Starts an AVD instance. + + Returns: + An _AvdInstance. + """ + instance = self.CreateInstance() + instance.Start() + return instance + + +class _AvdInstance(object): + """Represents a single running instance of an AVD. + + This class should only be created directly by AvdConfig.StartInstance, + but its other methods can be freely called. + """ + + def __init__(self, emulator_path, avd_name, emulator_home): + """Create an _AvdInstance object. + + Args: + emulator_path: path to the emulator binary. + avd_name: name of the AVD to run. + emulator_home: path to the emulator home directory. + """ + self._avd_name = avd_name + self._emulator_home = emulator_home + self._emulator_path = emulator_path + self._emulator_proc = None + self._emulator_serial = None + self._sink = None + + def __str__(self): + return '%s|%s' % (self._avd_name, (self._emulator_serial or id(self))) + + def Start(self, read_only=True, snapshot_save=False, window=False): + """Starts the emulator running an instance of the given AVD.""" + with tempfile_ext.TemporaryFileName() as socket_path, (contextlib.closing( + socket.socket(socket.AF_UNIX))) as sock: + sock.bind(socket_path) + emulator_cmd = [ + self._emulator_path, + '-avd', + self._avd_name, + '-report-console', + 'unix:%s' % socket_path, + ] + if read_only: + emulator_cmd.append('-read-only') + if not snapshot_save: + emulator_cmd.append('-no-snapshot-save') + emulator_env = {} + if self._emulator_home: + emulator_env['ANDROID_EMULATOR_HOME'] = self._emulator_home + if window: + if 'DISPLAY' in os.environ: + emulator_env['DISPLAY'] = os.environ.get('DISPLAY') + else: + raise AvdException('Emulator failed to start: DISPLAY not defined') + else: + emulator_cmd.append('-no-window') + sock.listen(1) + + logging.info('Starting emulator.') + + # TODO(jbudorick): Add support for logging emulator stdout & stderr at + # higher logging levels. + self._sink = open('/dev/null', 'w') + self._emulator_proc = cmd_helper.Popen( + emulator_cmd, stdout=self._sink, stderr=self._sink, env=emulator_env) + + # Waits for the emulator to report its serial as requested via + # -report-console. See http://bit.ly/2lK3L18 for more. + def listen_for_serial(s): + logging.info('Waiting for connection from emulator.') + with contextlib.closing(s.accept()[0]) as conn: + val = conn.recv(1024) + return 'emulator-%d' % int(val) + + try: + self._emulator_serial = timeout_retry.Run( + listen_for_serial, timeout=30, retries=0, args=[sock]) + logging.info('%s started', self._emulator_serial) + except Exception as e: + self.Stop() + raise AvdException('Emulator failed to start: %s' % str(e)) + + def Stop(self): + """Stops the emulator process.""" + if self._emulator_proc: + if self._emulator_proc.poll() is None: + self._emulator_proc.terminate() + self._emulator_proc.wait() + self._emulator_proc = None + if self._sink: + self._sink.close() + self._sink = None + + @property + def serial(self): + return self._emulator_serial diff --git a/chromium/build/android/pylib/local/emulator/local_emulator_environment.py b/chromium/build/android/pylib/local/emulator/local_emulator_environment.py index cd81cf9c3a7..22470c035e6 100644 --- a/chromium/build/android/pylib/local/emulator/local_emulator_environment.py +++ b/chromium/build/android/pylib/local/emulator/local_emulator_environment.py @@ -2,20 +2,14 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import contextlib import logging -import os -import socket -import stat -from py_utils import tempfile_ext - -from devil.android.sdk import adb_wrapper -from devil.utils import cmd_helper -from devil.utils import timeout_retry - -from pylib import constants +from devil.utils import parallelizer from pylib.local.device import local_device_environment +from pylib.local.emulator import avd + +# Mirroring https://bit.ly/2OjuxcS#23 +_MAX_ANDROID_EMULATORS = 16 class LocalEmulatorEnvironment(local_device_environment.LocalDeviceEnvironment): @@ -23,99 +17,52 @@ class LocalEmulatorEnvironment(local_device_environment.LocalDeviceEnvironment): def __init__(self, args, output_manager, error_func): super(LocalEmulatorEnvironment, self).__init__(args, output_manager, error_func) - self._avd_name = args.avd_name - self._emulator_home = (args.emulator_home - or os.path.expanduser(os.path.join('~', '.android'))) - - root_ini = os.path.join(self._emulator_home, 'avd', - '%s.ini' % self._avd_name) - if not os.path.exists(root_ini): - error_func('Unable to find configuration for AVD %s at %s' % - (self._avd_name, root_ini)) - - self._emulator_path = os.path.join(constants.ANDROID_SDK_ROOT, 'emulator', - 'emulator') - if not os.path.exists(self._emulator_path): - error_func('%s does not exist.' % self._emulator_path) - - self._emulator_proc = None - self._emulator_serial = None + self._avd_config = avd.AvdConfig(args.avd_config) + if args.emulator_count < 1: + error_func('--emulator-count must be >= 1') + elif args.emulator_count >= _MAX_ANDROID_EMULATORS: + logging.warning('--emulator-count capped at 16.') + self._emulator_count = min(_MAX_ANDROID_EMULATORS, args.emulator_count) + self._emulator_window = args.emulator_window + self._emulator_instances = [] + self._device_serials = [] #override def SetUp(self): - # Emulator start-up looks for the adb daemon. Make sure it's running. - adb_wrapper.AdbWrapper.StartServer() + self._avd_config.Install() - # Emulator start-up tries to check for the SDK root by looking for - # platforms/ and platform-tools/. Ensure they exist. - # See http://bit.ly/2YAkyFE for context. - required_dirs = [ - os.path.join(constants.ANDROID_SDK_ROOT, 'platforms'), - os.path.join(constants.ANDROID_SDK_ROOT, 'platform-tools'), + emulator_instances = [ + self._avd_config.CreateInstance() for _ in range(self._emulator_count) ] - for d in required_dirs: - if not os.path.exists(d): - os.makedirs(d) - # The emulator requires that some files are writable. - for dirname, _, filenames in os.walk(self._emulator_home): - for f in filenames: - path = os.path.join(dirname, f) - if (os.lstat(path).st_mode & - (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) == stat.S_IRUSR): - os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + def start_emulator_instance(e): + try: + e.Start(window=self._emulator_window) + return e + except avd.AvdException: + logging.exception('Failed to start emulator instance.') + return None + + parallel_emulators = parallelizer.SyncParallelizer(emulator_instances) + self._emulator_instances = [ + emu + for emu in parallel_emulators.pMap(start_emulator_instance).pGet(None) + if emu is not None + ] + self._device_serials = [e.serial for e in self._emulator_instances] - self._emulator_proc, self._emulator_serial = self._StartInstance() + if not self._emulator_instances: + raise Exception('Failed to start any instances of the emulator.') + elif len(self._emulator_instances) < self._emulator_count: + logging.warning( + 'Running with fewer emulator instances than requested (%d vs %d)', + len(self._emulator_instances), self._emulator_count) - logging.info('Emulator serial: %s', self._emulator_serial) - self._device_serials = [self._emulator_serial] super(LocalEmulatorEnvironment, self).SetUp() - def _StartInstance(self): - """Starts an AVD instance. - - Returns: - A (Popen, str) 2-tuple that includes the process and serial. - """ - # Start up the AVD. - with tempfile_ext.TemporaryFileName() as socket_path, (contextlib.closing( - socket.socket(socket.AF_UNIX))) as sock: - sock.bind(socket_path) - emulator_cmd = [ - self._emulator_path, - '-avd', - self._avd_name, - '-report-console', - 'unix:%s' % socket_path, - '-read-only', - '-no-window', - ] - emulator_env = {} - if self._emulator_home: - emulator_env['ANDROID_EMULATOR_HOME'] = self._emulator_home - sock.listen(1) - emulator_proc = cmd_helper.Popen(emulator_cmd, env=emulator_env) - - def listen_for_serial(s): - logging.info('Waiting for connection from emulator.') - with contextlib.closing(s.accept()[0]) as conn: - val = conn.recv(1024) - return 'emulator-%d' % int(val) - - try: - emulator_serial = timeout_retry.Run( - listen_for_serial, timeout=30, retries=0, args=[sock]) - except Exception: - emulator_proc.terminate() - raise - - return (emulator_proc, emulator_serial) - #override def TearDown(self): try: super(LocalEmulatorEnvironment, self).TearDown() finally: - if self._emulator_proc: - self._emulator_proc.terminate() - self._emulator_proc.wait() + parallelizer.SyncParallelizer(self._emulator_instances).Stop() diff --git a/chromium/build/android/pylib/local/emulator/proto/__init__.py b/chromium/build/android/pylib/local/emulator/proto/__init__.py new file mode 100644 index 00000000000..4a12e35c925 --- /dev/null +++ b/chromium/build/android/pylib/local/emulator/proto/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/chromium/build/android/pylib/local/emulator/proto/avd.proto b/chromium/build/android/pylib/local/emulator/proto/avd.proto new file mode 100644 index 00000000000..adf5cb76469 --- /dev/null +++ b/chromium/build/android/pylib/local/emulator/proto/avd.proto @@ -0,0 +1,35 @@ + +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto3"; + +package tools.android.avd.proto; + +message CIPDPackage { + // CIPD package name. + string package_name = 1; + // CIPD package version to use. + // Ignored when creating AVD packages. + string version = 2; + // Path into which the package should be installed. + // src-relative. + string dest_path = 3; +} + +message Avd { + // The emulator to use in running the AVD. + CIPDPackage emulator_package = 1; + + // The system image to use. + CIPDPackage system_image_package = 2; + // The name of the system image to use, as reported by sdkmanager. + string system_image_name = 3; + + // The AVD to create or use. + // (Only the package_name is used during AVD creation.) + CIPDPackage avd_package = 4; + // The name of the AVD to create or use. + string avd_name = 5; +} diff --git a/chromium/build/android/pylib/local/emulator/proto/avd_pb2.py b/chromium/build/android/pylib/local/emulator/proto/avd_pb2.py new file mode 100644 index 00000000000..c264e6d17fe --- /dev/null +++ b/chromium/build/android/pylib/local/emulator/proto/avd_pb2.py @@ -0,0 +1,218 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: avd.proto + +import sys +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + +DESCRIPTOR = _descriptor.FileDescriptor( + name='avd.proto', + package='tools.android.avd.proto', + syntax='proto3', + serialized_pb=_b( + '\n\tavd.proto\x12\x17tools.android.avd.proto\"G\n\x0b\x43IPDPackage\x12\x14\n\x0cpackage_name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x11\n\tdest_path\x18\x03 \x01(\t\"\xf1\x01\n\x03\x41vd\x12>\n\x10\x65mulator_package\x18\x01 \x01(\x0b\x32$.tools.android.avd.proto.CIPDPackage\x12\x42\n\x14system_image_package\x18\x02 \x01(\x0b\x32$.tools.android.avd.proto.CIPDPackage\x12\x19\n\x11system_image_name\x18\x03 \x01(\t\x12\x39\n\x0b\x61vd_package\x18\x04 \x01(\x0b\x32$.tools.android.avd.proto.CIPDPackage\x12\x10\n\x08\x61vd_name\x18\x05 \x01(\tb\x06proto3' + )) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +_CIPDPACKAGE = _descriptor.Descriptor( + name='CIPDPackage', + full_name='tools.android.avd.proto.CIPDPackage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='package_name', + full_name='tools.android.avd.proto.CIPDPackage.package_name', + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='version', + full_name='tools.android.avd.proto.CIPDPackage.version', + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='dest_path', + full_name='tools.android.avd.proto.CIPDPackage.dest_path', + index=2, + number=3, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + ], + extensions=[], + nested_types=[], + enum_types=[], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=38, + serialized_end=109, +) + +_AVD = _descriptor.Descriptor( + name='Avd', + full_name='tools.android.avd.proto.Avd', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='emulator_package', + full_name='tools.android.avd.proto.Avd.emulator_package', + index=0, + number=1, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='system_image_package', + full_name='tools.android.avd.proto.Avd.system_image_package', + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='system_image_name', + full_name='tools.android.avd.proto.Avd.system_image_name', + index=2, + number=3, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='avd_package', + full_name='tools.android.avd.proto.Avd.avd_package', + index=3, + number=4, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='avd_name', + full_name='tools.android.avd.proto.Avd.avd_name', + index=4, + number=5, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + ], + extensions=[], + nested_types=[], + enum_types=[], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=112, + serialized_end=353, +) + +_AVD.fields_by_name['emulator_package'].message_type = _CIPDPACKAGE +_AVD.fields_by_name['system_image_package'].message_type = _CIPDPACKAGE +_AVD.fields_by_name['avd_package'].message_type = _CIPDPACKAGE +DESCRIPTOR.message_types_by_name['CIPDPackage'] = _CIPDPACKAGE +DESCRIPTOR.message_types_by_name['Avd'] = _AVD + +CIPDPackage = _reflection.GeneratedProtocolMessageType( + 'CIPDPackage', + (_message.Message, ), + dict( + DESCRIPTOR=_CIPDPACKAGE, + __module__='avd_pb2' + # @@protoc_insertion_point(class_scope:tools.android.avd.proto.CIPDPackage) + )) +_sym_db.RegisterMessage(CIPDPackage) + +Avd = _reflection.GeneratedProtocolMessageType( + 'Avd', + (_message.Message, ), + dict( + DESCRIPTOR=_AVD, + __module__='avd_pb2' + # @@protoc_insertion_point(class_scope:tools.android.avd.proto.Avd) + )) +_sym_db.RegisterMessage(Avd) + +# @@protoc_insertion_point(module_scope) diff --git a/chromium/build/android/pylib/local/machine/local_machine_junit_test_run.py b/chromium/build/android/pylib/local/machine/local_machine_junit_test_run.py index 312bf9c6ff9..dab18e32000 100644 --- a/chromium/build/android/pylib/local/machine/local_machine_junit_test_run.py +++ b/chromium/build/android/pylib/local/machine/local_machine_junit_test_run.py @@ -5,6 +5,7 @@ import json import logging import os +import zipfile from devil.utils import cmd_helper from pylib import constants @@ -31,7 +32,6 @@ class LocalMachineJunitTestRun(test_run.TestRun): def RunTests(self, results): with tempfile_ext.NamedTemporaryDirectory() as temp_dir: json_file_path = os.path.join(temp_dir, 'results.json') - java_script = os.path.join( constants.GetOutDirectory(), 'bin', 'helper', self._test_instance.suite) @@ -55,8 +55,6 @@ class LocalMachineJunitTestRun(test_run.TestRun): self._test_instance.robolectric_runtime_deps_dir, '-Ddir.source.root=%s' % constants.DIR_SOURCE_ROOT, '-Drobolectric.resourcesMode=binary', - '-Dchromium.robolectric.resource.ap_=%s' % - self._test_instance.resource_apk ] if logging.getLogger().isEnabledFor(logging.INFO): @@ -90,6 +88,14 @@ class LocalMachineJunitTestRun(test_run.TestRun): if jvm_args: command.extend(['--jvm-args', '"%s"' % ' '.join(jvm_args)]) + # Create properties file for Robolectric test runners so they can find the + # binary resources. + properties_jar_path = os.path.join(temp_dir, 'properties.jar') + with zipfile.ZipFile(properties_jar_path, 'w') as z: + z.writestr('com/android/tools/test_config.properties', + 'android_resource_apk=%s' % self._test_instance.resource_apk) + command.extend(['--classpath', properties_jar_path]) + cmd_helper.RunCmd(command) try: with open(json_file_path, 'r') as f: |