diff options
Diffstat (limited to 'scripts/computerankings.py')
| -rwxr-xr-x | scripts/computerankings.py | 359 |
1 files changed, 0 insertions, 359 deletions
diff --git a/scripts/computerankings.py b/scripts/computerankings.py deleted file mode 100755 index 8ad1fd3..0000000 --- a/scripts/computerankings.py +++ /dev/null @@ -1,359 +0,0 @@ -#!/usr/bin/env python - -import sys -from dbaccess import setDatabase, execQuery, commit -from misc import ( - getOptions, textToId, getAllSnapshots, getLastRankingSnapshot, getContext, - isValidSHA1, getBMTimeSeriesStatsList) - - -# --- BEGIN Global functions ---------------------------------------------- - -def printUsage(): - sys.stderr.write( - "usage: " + sys.argv[0] + - " --help | [--dbhost H] [--dbport P] --db D --host H --platform P " - "--branch B --sha1 S [--noprogress NP]\n") - -def printVerboseUsage(): - printUsage() - sys.stderr.write("\noptions:\n") - sys.stderr.write( - " --help: This help.\n") - sys.stderr.write( - " --dbhost: The database server host (overriding the default).\n") - sys.stderr.write( - " --dbport: The database server port (overriding the default).\n") - sys.stderr.write( - " --db: The database. One of 'bm' or 'bm-dev' (the latter " - "intended for experimentation).\n") - sys.stderr.write( - " --host: The physical machine on which the results were " - "produced (e.g. barbarella or 172.24.90.79).\n") - sys.stderr.write( - "--platform: The OS/compiler/architecture combination " - "(e.g. linux-g++-32).\n") - sys.stderr.write( - " --branch: The product branch (e.g. 'qt 4.6', 'qt 4.7', or " - "'qt master').\n") - sys.stderr.write( - " --sha1: The tested revision within the branch. Can be " - "extracted using 'git log -1 --pretty=format:%H' (assuming the " - "tested revision is the current head revision).\n") - sys.stderr.write( - " --noprogress: Specify \'true\' to disable progress indicator.\n") - - -# ### 2 B DOCUMENTED! -def printProgress(p, lead): - sys.stdout.write(lead + " ... (" + "{0:.2f}".format(p) + " %)\r") - sys.stdout.flush() - - -# ### 2 B DOCUMENTED! -# NOTE: This function is currently duplicated elsewhere in JavaScipt! -def changeMagnitudeScore(change): - max_change = 2.0 - abs_change = (1.0 / change) if change < 1 else change - return (min(abs_change, max_change) - 1.0) / (max_change - 1.0) - - -# ### 2 B DOCUMENTED! -# NOTE: This function is currently duplicated elsewhere in JavaScript! -def qualityScore(lsd, ni, nz, nc, mdrse): - max_bad_snapshots = 10 # experimental; maybe use max durability score? - max_sample_size = 5; - max_LSD = max_bad_snapshots; - max_NI = max_bad_snapshots * max_sample_size; - max_NZ = max_bad_snapshots * max_sample_size; - max_NC = max_bad_snapshots; - - lsd_score = 0 if (lsd == -1) else min(1, lsd / float(max_LSD)); - ni_score = min(1, ni / float(max_NI)); - nz_score = min(1, nz / float(max_NZ)); - nc_score = min(1, nc / float(max_NC)); - mdrse_score = 0 if (mdrse == -1) else (mdrse / 100.0); - - return (lsd_score + ni_score + nz_score + nc_score + mdrse_score) / 5.0; - - -# Registers the ranking for a given statistic. context1_id and context2_id -# refer to the first and last snapshot respectively in the interval used for -# computing the rankings. -# Assumptions: -# - A high value should be ranked above a small one. -# - A negative value is undefined and gets an invalid ranking position, i.e. -1. -def registerRanking(table, stat_index, stat_name, context1_id, context2_id): - - table.sort(key=lambda x: x[stat_index], reverse=True) - - stat_id = textToId("rankingStat", stat_name) - assert stat_id >= 0 - - row_pos = 0 - ranking_pos = 0 - for row in table: - benchmark_id = row[0] - metric_id = row[1] - lc_timestamp = row[2] - stat_value = row[stat_index] - - # The following statement ensures the following conditions: - # - A negative value gets an invalid ranking position, i.e. -1 - # - Equal values get the same ranking position. - # - The ranking position of benchmark B indicates the number of - # benchmarks ranked higher than B (i.e. having a smaller ranking - # position). - if stat_value < 0: - ranking_pos = -1 - # Note that the remaining values will now be negative, so updating - # row_pos and prev_stat_value is no longer necessary! - else: - if (row_pos > 0) and (stat_value != prev_stat_value): - ranking_pos = row_pos - row_pos = row_pos + 1 - prev_stat_value = stat_value - - # Insert or update the corresponding row in the 'ranking' table: - execQuery( - "SELECT merge_ranking(%s, %s, %s, %s, %s, %s, %s, %s)", - (context1_id, context2_id, benchmark_id, metric_id, - lc_timestamp, stat_id, stat_value, ranking_pos), - False) - - -# ### 2 B DOCUMENTED! -def getAllRankingStats(bmstats_list): - table = [] - for stats in bmstats_list: - - # NOTE: - # - All of the ranking statistics are of type "higher is better" - # (a high value is ranked better than a low value). - # - Moreover, all present/defined values are non-negative. - # - This means that representing absent/undefined values as -1 is ok, - # since this ensures lowest ranking. - - benchmark_id = stats["benchmark_id"] - metric_id = stats["metric_id"] - lc_timestamp = stats["lc_timestamp"] - lsd = stats["lsd"] - ni = stats["ni"] - nz = stats["nz"] - nc = stats["nc"] - mdrse = stats["med_of_rses"] - rsemd = stats["rse_of_meds"] - - qs = qualityScore(lsd, ni, nz, nc, mdrse) - - lc = stats["lc"] - if lc >= 0.0: - lcgss = stats["lc_gsep_score"] - lclss = stats["lc_lsep_score"] - lcds1 = stats["lc_dur1_score"] - lcds2 = stats["lc_dur2_score"] - lcms = changeMagnitudeScore(lc) - lcss1 = lcms * lcgss * lclss * lcds1 - lcss = lcss1 * lcds2 - if lc < 1.0: - lcssr = lcss - lcss1r = lcss1 - lcssi = lcss1i = -1 - else: - lcssi = lcss - lcss1i = lcss1 - lcssr = lcss1r = -1 - else: - lcssr = lcssi = lcss1r = lcss1i = -1 - - table.append( - (benchmark_id, metric_id, lc_timestamp, qs, lcssr, lcssi, lcss1r, - lcss1i)) - - return table - - -# ### 2 B DOCUMENTED! -def getFirstUploadTimestamp(snapshots, sha1_id): - try: - return snapshots[zip(*snapshots)[0].index(sha1_id)][1] - except ValueError: - return -1 - - -# ### 2 B DOCUMENTED! -def updateRankings( - host_id, platform_id, branch_id, sha12_id, context2_id, no_progress): - - # Get all snapshots matching the host/platform/branch combination: - sys.stdout.write("getting snapshots ... ") - sys.stdout.flush() - snapshots = getAllSnapshots(host_id, platform_id, branch_id) - sys.stdout.write("done\n") - sys.stdout.flush() - - - # Rankings will normally be computed once a day for each - # host/platform/branch combination (note the tradeoff between update - # frequency and database size): - ranking_interval = 3600 * 24 # secs in a day - - # Rankings will be updated if at least one of the following - # conditions eventually becomes True: - force_cond = empty_cond = interval_cond = False - - force_ranking = False - #force_ranking = True # Uncomment for testing - - force_cond = force_ranking - - if not force_cond: - last_ranking_sha1_id, last_ranking_timestamp = getLastRankingSnapshot( - host_id, platform_id, branch_id) - empty_cond = last_ranking_sha1_id < 0 - if not empty_cond: - assert last_ranking_timestamp >= 0 - - target_timestamp = getFirstUploadTimestamp(snapshots, sha12_id) - if target_timestamp < 0: - sys.stderr.write( - "error: failed to extract target_timestamp " - "(error in command-line args?)\n") - sys.exit(1) - - interval_cond = ( - (target_timestamp - last_ranking_timestamp) > ranking_interval) - - if not (force_cond or empty_cond or interval_cond): - sys.stdout.write( - "not updating rankings ('force', 'empty', and 'interval' " - "conditions all failed)\n") - return - - sys.stdout.write( - "updating rankings ('force' cond.: " + str(force_cond) + - "; 'empty' cond.: " + str(empty_cond) + - "; 'interval' cond.: " + str(interval_cond) + ") ...\n") - - # For simplicity we hardcode the tolerances for now: - difftol = 1.1 - durtolmin = 3 - durtolmax = 10 - - # Determine the target snapshot range: - # (The range should end at the snapshot given on the command-line and begin - # at the snapshot that is 2 * durtolmax snapshots back in time, or, if no - # such snapshot exists, the first available snapshot.) - try: - sha12_pos = zip(*snapshots)[0].index(sha12_id) - except ValueError: - sys.stderr.write( - "no observations found for SHA-1 ID: " + str(sha12_id) + "\n") - sys.exit(1) - sha11_pos = max(0, (sha12_pos - 2 * durtolmax) + 1) - snapshots = snapshots[sha11_pos:(sha12_pos + 1)] - if len(snapshots) < 2: - sys.stderr.write( - "no observations found before SHA-1 ID: " + str(sha12_id) + - " (computing rankings makes no sense)\n") - sys.exit(1) - - # Get time series statistics for all benchmarks: - if no_progress: - sys.stdout.write("getting time series statistics ... ") - bmstats_list = getBMTimeSeriesStatsList( - host_id, platform_id, branch_id, snapshots, None, difftol, durtolmin, - durtolmax, None if no_progress else printProgress, - "getting time series statistics") - - if no_progress: - sys.stdout.write("done\n") - else: - sys.stdout.write("\n") - - - # *** Compute rankings ************************************************** - - # Step 1: Create a table containing all ranking statistics (one row per - # benchmark/metric): - sys.stdout.write("creating table for all ranking stats ... ") - sys.stdout.flush() - table = getAllRankingStats(bmstats_list) - sys.stdout.write("done\n") - sys.stdout.flush() - - # Step 2: Sort the table individually for each ranking statistic and - # register the ranking positions in the database: - context1_id = getContext(host_id, platform_id, branch_id, snapshots[0][0]) - if context1_id == -1: - sys.stderr.write("error: failed to find context for start snapshot\n") - sys.exit(1) - nameToIndex = { "QS": 3, "LCSSR": 4, "LCSSI": 5, "LCSS1R": 6, "LCSS1I": 7 } - for name in nameToIndex: - sys.stdout.write("registering ranking for " + name + " ... ") - sys.stdout.flush() - registerRanking( - table, nameToIndex[name], name, context1_id, context2_id) - sys.stdout.write("done\n") - sys.stdout.flush() - -# --- END Global functions ---------------------------------------------- - - -# --- BEGIN Main program ---------------------------------------------- - -options, http_get = getOptions() - -if "help" in options: - printVerboseUsage() - sys.exit(1) - -if (not ("db" in options and "host" in options and "platform" in options and - "branch" in options and "sha1" in options)): - printUsage() - sys.exit(1) - -if not isValidSHA1(options["sha1"]): - sys.stderr.write("error: invalid SHA-1: " + options["sha1"] + "\n") - sys.exit(1) - -setDatabase( - options["dbhost"] if "dbhost" in options else None, - options["dbport"] if "dbport" in options else None, - options["db"]) - -host_id = textToId("host", options["host"]) -if host_id == -1: - sys.stderr.write("error: no such host: " + options["host"] + "\n") - sys.exit(1) -platform_id = textToId("platform", options["platform"]) -if platform_id == -1: - sys.stderr.write("error: no such platform: " + options["platform"] + "\n") - sys.exit(1) -branch_id = textToId("branch", options["branch"]) -if branch_id == -1: - sys.stderr.write("error: no such branch:" + options["branch"] + "\n") - sys.exit(1) -sha12_id = textToId("sha1", options["sha1"]) -if sha12_id == -1: - sys.stderr.write("error: no such SHA-1:" + options["sha1"] + "\n") - sys.exit(1) - -context2_id = getContext(host_id, platform_id, branch_id, sha12_id) -if context2_id == -1: - sys.stderr.write("error: no results found for this context\n") - sys.exit(1) - -updateRankings( - host_id, platform_id, branch_id, sha12_id, context2_id, - ("noprogress" in options) and ( - (options["noprogress"] == "1") - or (options["noprogress"].lower() == "true"))) - -# Write to database: -commit() - -sys.stdout.write("rankings computation done\n") -sys.exit(0) - -# --- END Main program ---------------------------------------------- |