diff options
Diffstat (limited to 'chromium/third_party/skia/bench/gen_bench_expectations.py')
| -rw-r--r-- | chromium/third_party/skia/bench/gen_bench_expectations.py | 223 |
1 files changed, 223 insertions, 0 deletions
diff --git a/chromium/third_party/skia/bench/gen_bench_expectations.py b/chromium/third_party/skia/bench/gen_bench_expectations.py new file mode 100644 index 00000000000..4edc38c09d2 --- /dev/null +++ b/chromium/third_party/skia/bench/gen_bench_expectations.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +# Copyright (c) 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" Generate bench_expectations file from a given set of bench data files. """ + +import argparse +import bench_util +import json +import os +import re +import sys +import urllib2 + +# Parameters for calculating bench ranges. +RANGE_RATIO_UPPER = 1.5 # Ratio of range for upper bounds. +RANGE_RATIO_LOWER = 2.0 # Ratio of range for lower bounds. +ERR_RATIO = 0.08 # Further widens the range by the ratio of average value. +ERR_UB = 1.0 # Adds an absolute upper error to cope with small benches. +ERR_LB = 1.5 + +# List of bench configs to monitor. Ignore all other configs. +CONFIGS_TO_INCLUDE = ['simple_viewport_1000x1000', + 'simple_viewport_1000x1000_angle', + 'simple_viewport_1000x1000_gpu', + 'simple_viewport_1000x1000_scalar_1.100000', + 'simple_viewport_1000x1000_scalar_1.100000_gpu', + ] + +# List of flaky entries that should be excluded. Each entry is defined by a list +# of 3 strings, corresponding to the substrings of [bench, config, builder] to +# search for. A bench expectations line is excluded when each of the 3 strings +# in the list is a substring of the corresponding element of the given line. For +# instance, ['desk_yahooanswers', 'gpu', 'Ubuntu'] will skip expectation entries +# of SKP benchs whose name contains 'desk_yahooanswers' on all gpu-related +# configs of all Ubuntu builders. +ENTRIES_TO_EXCLUDE = [ + ] + +_GS_CLOUD_FORMAT = 'http://storage.googleapis.com/chromium-skia-gm/perfdata/%s/%s' + +def compute_ranges(benches, more_benches=None): + """Given a list of bench numbers, calculate the alert range. + + Args: + benches: a list of float bench values. + more_benches: a tuple of lists of additional bench values. + The first value of each tuple is the number of commits before the current + one that set of values is at, and the second value is a list of + bench results. + + Returns: + a list of float [lower_bound, upper_bound]. + """ + avg = sum(benches)/len(benches) + minimum = min(benches) + maximum = max(benches) + diff = maximum - minimum + + return [minimum - diff*RANGE_RATIO_LOWER - avg*ERR_RATIO - ERR_LB, + maximum + diff*RANGE_RATIO_UPPER + avg*ERR_RATIO + ERR_UB] + + +def create_expectations_dict(revision_data_points, builder, extra_data=None): + """Convert list of bench data points into a dictionary of expectations data. + + Args: + revision_data_points: a list of BenchDataPoint objects. + builder: string of the corresponding buildbot builder name. + + Returns: + a dictionary of this form: + keys = tuple of (config, bench) strings. + values = list of float [expected, lower_bound, upper_bound] for the key. + """ + bench_dict = {} + for point in revision_data_points: + if (point.time_type or # Not walltime which has time_type '' + not point.config in CONFIGS_TO_INCLUDE): + continue + to_skip = False + for bench_substr, config_substr, builder_substr in ENTRIES_TO_EXCLUDE: + if (bench_substr in point.bench and config_substr in point.config and + builder_substr in builder): + to_skip = True + break + if to_skip: + continue + key = (point.config, point.bench) + + extras = [] + for idx, dataset in extra_data: + for data in dataset: + if (data.bench == point.bench and data.config == point.config and + data.time_type == point.time_type and data.per_iter_time): + extras.append((idx, data.per_iter_time)) + + if key in bench_dict: + raise Exception('Duplicate bench entry: ' + str(key)) + bench_dict[key] = [point.time] + compute_ranges(point.per_iter_time, extras) + + return bench_dict + + +def get_parent_commits(start_hash, num_back): + """Returns a list of commits that are the parent of the commit passed in.""" + list_commits = urllib2.urlopen( + 'https://skia.googlesource.com/skia/+log/%s?format=json&n=%d' % + (start_hash, num_back)) + # NOTE: Very brittle. Removes the four extraneous characters + # so json can be read successfully + trunc_list = list_commits.read()[4:] + json_data = json.loads(trunc_list) + return [revision['commit'] for revision in json_data['log']] + + +def get_file_suffixes(commit_hash, directory): + """Gets all the suffixes available in the directory""" + possible_files = os.listdir(directory) + prefix = 'bench_' + commit_hash + '_data_' + return [name[len(prefix):] for name in possible_files + if name.startswith(prefix)] + + +def download_bench_data(builder, commit_hash, suffixes, directory): + """Downloads data, returns the number successfully downloaded""" + cur_files = os.listdir(directory) + count = 0 + for suffix in suffixes: + file_name = 'bench_'+commit_hash+'_data_'+suffix + if file_name in cur_files: + continue + try: + src = urllib2.urlopen(_GS_CLOUD_FORMAT % (builder, file_name)) + with open(os.path.join(directory, file_name), 'w') as dest: + dest.writelines(src) + count += 1 + except urllib2.HTTPError: + pass + return count + + +def main(): + """Reads bench data points, then calculate and export expectations. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + '-a', '--representation_alg', default='25th', + help='bench representation algorithm to use, see bench_util.py.') + parser.add_argument( + '-b', '--builder', required=True, + help='name of the builder whose bench ranges we are computing.') + parser.add_argument( + '-d', '--input_dir', required=True, + help='a directory containing bench data files.') + parser.add_argument( + '-o', '--output_file', required=True, + help='file path and name for storing the output bench expectations.') + parser.add_argument( + '-r', '--git_revision', required=True, + help='the git hash to indicate the revision of input data to use.') + parser.add_argument( + '-t', '--back_track', required=False, default=10, + help='the number of commit hashes backwards to look to include' + + 'in the calculations.') + parser.add_argument( + '-m', '--max_commits', required=False, default=1, + help='the number of commit hashes to include in the calculations.') + args = parser.parse_args() + + builder = args.builder + + data_points = bench_util.parse_skp_bench_data( + args.input_dir, args.git_revision, args.representation_alg) + + parent_commits = get_parent_commits(args.git_revision, args.back_track) + print "Using commits: {}".format(parent_commits) + suffixes = get_file_suffixes(args.git_revision, args.input_dir) + print "Using suffixes: {}".format(suffixes) + + # TODO(kelvinly): Find a better approach to than directly copying from + # the GS server? + downloaded_commits = [] + for idx, commit in enumerate(parent_commits): + num_downloaded = download_bench_data( + builder, commit, suffixes, args.input_dir) + if num_downloaded > 0: + downloaded_commits.append((num_downloaded, idx, commit)) + + if len(downloaded_commits) < args.max_commits: + print ('Less than desired number of commits found. Please increase' + '--back_track in later runs') + trunc_commits = sorted(downloaded_commits, reverse=True)[:args.max_commits] + extra_data = [] + for _, idx, commit in trunc_commits: + extra_data.append((idx, bench_util.parse_skp_bench_data( + args.input_dir, commit, args.representation_alg))) + + expectations_dict = create_expectations_dict(data_points, builder, + extra_data) + + out_lines = [] + keys = expectations_dict.keys() + keys.sort() + for (config, bench) in keys: + (expected, lower_bound, upper_bound) = expectations_dict[(config, bench)] + out_lines.append('%(bench)s_%(config)s_,%(builder)s-%(representation)s,' + '%(expected)s,%(lower_bound)s,%(upper_bound)s' % { + 'bench': bench, + 'config': config, + 'builder': builder, + 'representation': args.representation_alg, + 'expected': expected, + 'lower_bound': lower_bound, + 'upper_bound': upper_bound}) + + with open(args.output_file, 'w') as file_handle: + file_handle.write('\n'.join(out_lines)) + + +if __name__ == "__main__": + main() |