From f391632c21afb4d122cd6e22c2de967257faddde Mon Sep 17 00:00:00 2001 From: jasplin Date: Thu, 10 Mar 2011 15:24:09 +0100 Subject: Let uploadresults.py execute computerankings.py itself as appropriate. --- scripts/computerankings.py | 365 +++++++++++++++++++++++++++++++++++++++++++++ scripts/finalizeresults.py | 353 ------------------------------------------- scripts/uploadresults.py | 97 +++++++++++- 3 files changed, 454 insertions(+), 361 deletions(-) create mode 100755 scripts/computerankings.py delete mode 100755 scripts/finalizeresults.py diff --git a/scripts/computerankings.py b/scripts/computerankings.py new file mode 100755 index 0000000..6726806 --- /dev/null +++ b/scripts/computerankings.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python + +import sys +from dbaccess import setDatabase, execQuery, commit +from misc import ( + getOptions, textToId, getAllSnapshots, getLastRankingSnapshot, getContext, + isValidSHA1, getBMTimeSeriesStatsList) + + +# --- BEGIN Global functions ---------------------------------------------- + +def printUsage(): + print ( + "usage:" + sys.argv[0] + + " --help | [--dbhost H] [--dbport P] --db D --host H --platform P " + "--branch B --sha1 S [--noprogress NP]") + +def printVerboseUsage(): + printUsage() + print "\noptions:" + print( + " --help: This help.") + print( + " --dbhost: The database server host (overriding the default).") + print( + " --dbport: The database server port (overriding the default).") + print( + " --db: The database. One of 'bm' or 'bm-dev' (the latter " + "intended for experimentation).") + print( + " --host: The physical machine on which the results were " + "produced (e.g. barbarella or 172.24.90.79).") + print( + "--platform: The OS/compiler/architecture combination " + "(e.g. linux-g++-32).") + print( + " --branch: The product branch (e.g. 'qt 4.6', 'qt 4.7', or " + "'qt master').") + print( + " --sha1: The tested revision within the branch. Can be " + "extracted using 'git log -1 --pretty=format:%H' (assuming the " + "tested revision is the current head revision).") + print( + " --noprogress: Specify \'true\' to disable progress indicator.") + + +# ### 2 B DOCUMENTED! +def printProgress(p, lead): + sys.stdout.write(lead + " ... (" + "{0:.2f}".format(p) + " %)\r") + sys.stdout.flush() + + +# ### 2 B DOCUMENTED! +# NOTE: This function is currently duplicated elsewhere in JavaScipt! +def changeMagnitudeScore(change): + max_change = 2.0 + abs_change = (1.0 / change) if change < 1 else change + return (min(abs_change, max_change) - 1.0) / (max_change - 1.0) + + +# ### 2 B DOCUMENTED! +# NOTE: This function is currently duplicated elsewhere in JavaScript! +def qualityScore(lsd, ni, nz, nc, mdrse): + max_bad_snapshots = 30 # experimental; maybe use max durability score? + max_sample_size = 5; + max_LSD = max_bad_snapshots; + max_NI = max_bad_snapshots * max_sample_size; + max_NZ = max_bad_snapshots * max_sample_size; + max_NC = max_bad_snapshots; + + lsd_score = 0 if (lsd == -1) else min(1, lsd / float(max_LSD)); + ni_score = min(1, ni / float(max_NI)); + nz_score = min(1, nz / float(max_NZ)); + nc_score = min(1, nc / float(max_NC)); + mdrse_score = 0 if (mdrse == -1) else (mdrse / 100.0); + + return (lsd_score + ni_score + nz_score + nc_score + mdrse_score) / 5.0; + + +# Registers the ranking for a given statistic. context1_id and context2_id +# refer to the first and last snapshot respectively in the interval used for +# computing the rankings. +# Assumptions: +# - A high value should be ranked above a small one. +# - A negative value is undefined and gets an invalid ranking position, i.e. -1. +def registerRanking(table, stat_index, stat_name, context1_id, context2_id): + + table.sort(key=lambda x: x[stat_index], reverse=True) + + stat_id = textToId("rankingStat", stat_name) + assert stat_id >= 0 + + row_pos = 0 + ranking_pos = 0 + for row in table: + benchmark_id = row[0] + metric_id = row[1] + lc_timestamp = row[2] + stat_value = row[stat_index] + + # The following statement ensures the following conditions: + # - A negative value gets an invalid ranking position, i.e. -1 + # - Equal values get the same ranking position. + # - The ranking position of benchmark B indicates the number of + # benchmarks ranked higher than B (i.e. having a smaller ranking + # position). + if stat_value < 0: + ranking_pos = -1 + # Note that the remaining values will now be negative, so updating + # row_pos and prev_stat_value is no longer necessary! + else: + if (row_pos > 0) and (stat_value != prev_stat_value): + ranking_pos = row_pos + row_pos = row_pos + 1 + prev_stat_value = stat_value + + # Insert or update the corresponding row in the 'ranking' table: + query = ( + "SELECT merge_ranking(" + + str(context1_id) + + ", " + str(context2_id) + + ", " + str(benchmark_id) + + ", " + str(metric_id) + + ", " + str(lc_timestamp) + + ", " + str(stat_id) + + ", " + str(stat_value) + + ", " + str(ranking_pos) + + ");" + ) + execQuery(query, False) + + +# ### 2 B DOCUMENTED! +def getAllRankingStats(bmstats_list): + table = [] + for stats in bmstats_list: + + # NOTE: + # - All of the ranking statistics are of type "higher is better" + # (a high value is ranked better than a low value). + # - Moreover, all present/defined values are non-negative. + # - This means that representing absent/undefined values as -1 is ok, + # since this ensures lowest ranking. + + benchmark_id = stats["benchmark_id"] + metric_id = stats["metric_id"] + lc_timestamp = stats["lc_timestamp"] + lsd = stats["lsd"] + ni = stats["ni"] + nz = stats["nz"] + nc = stats["nc"] + mdrse = stats["med_of_rses"] + rsemd = stats["rse_of_meds"] + + qs = qualityScore(lsd, ni, nz, nc, mdrse) + + lc = stats["lc"] + if lc >= 0.0: + lcgss = stats["lc_gsep_score"] + lclss = stats["lc_lsep_score"] + lcds1 = stats["lc_dur1_score"] + lcds2 = stats["lc_dur2_score"] + lcms = changeMagnitudeScore(lc) + lcss1 = (lcms + lcgss + lclss + lcds1) / 4.0 + lcss = (lcms + lcgss + lclss + lcds1 + lcds2) / 5.0 + if lc < 1.0: + lcssr = lcss + lcss1r = lcss1 + lcssi = lcss1i = -1 + else: + lcssi = lcss + lcss1i = lcss1 + lcssr = lcss1r = -1 + else: + lcssr = lcssi = lcss1r = lcss1i = -1 + + table.append( + (benchmark_id, metric_id, lc_timestamp, qs, lcssr, lcssi, lcss1r, + lcss1i)) + + return table + + +# ### 2 B DOCUMENTED! +def getFirstUploadTimestamp(snapshots, sha1_id): + try: + return snapshots[zip(*snapshots)[0].index(sha1_id)][1] + except ValueError: + return -1 + + +# ### 2 B DOCUMENTED! +def updateRankings( + host_id, platform_id, branch_id, sha12_id, context2_id, no_progress): + + # Get all snapshots matching the host/platform/branch combination: + sys.stdout.write("getting snapshots ... ") + sys.stdout.flush() + snapshots = getAllSnapshots(host_id, platform_id, branch_id) + sys.stdout.write("done\n") + sys.stdout.flush() + + + # Rankings will normally be computed once a day for each + # host/platform/branch combination (note the tradeoff between update + # frequency and database size): + ranking_interval = 3600 * 24 # secs in a day + + # Rankings will be updated if at least one of the following + # conditions eventually becomes True: + force_cond = empty_cond = interval_cond = False + + force_ranking = False + #force_ranking = True # Uncomment for testing + + force_cond = force_ranking + + if not force_cond: + last_ranking_sha1_id, last_ranking_timestamp = getLastRankingSnapshot( + host_id, platform_id, branch_id) + empty_cond = last_ranking_sha1_id < 0 + if not empty_cond: + assert last_ranking_timestamp >= 0 + + target_timestamp = getFirstUploadTimestamp(snapshots, sha12_id) + if target_timestamp < 0: + print ( + "error: failed to extract target_timestamp " + "(error in command-line args?)") + sys.exit(1) + + interval_cond = ( + (target_timestamp - last_ranking_timestamp) > ranking_interval) + + if not (force_cond or empty_cond or interval_cond): + print ( + "not updating rankings ('force', 'empty', and 'interval' " + "conditions all failed)") + return + + print ( + "updating rankings ('force' cond.: " + str(force_cond) + + "; 'empty' cond.: " + str(empty_cond) + + "; 'interval' cond.: " + str(interval_cond) + ") ...") + + # For simplicity we hardcode the tolerances for now: + difftol = 1.1 + durtolmin = 3 + durtolmax = 30 + + # Determine the target snapshot range: + # (The range should end at the snapshot given on the command-line and begin + # at the snapshot that is 2 * durtolmax snapshots back in time, or, if no + # such snapshot exists, the first available snapshot.) + try: + sha12_pos = zip(*snapshots)[0].index(sha12_id) + except ValueError: + print "no observations found for SHA-1 ID:", sha12_id + sys.exit(1) + sha11_pos = max(0, (sha12_pos - 2 * durtolmax) + 1) + snapshots = snapshots[sha11_pos:(sha12_pos + 1)] + if len(snapshots) < 2: + print ( + "no observations found before SHA-1 ID: " + sha12_id + + " (computing rankings makes no sense)") + sys.exit(1) + + # Get time series statistics for all benchmarks: + if no_progress: + sys.stdout.write("getting time series statistics ... ") + bmstats_list = getBMTimeSeriesStatsList( + host_id, platform_id, branch_id, snapshots, None, difftol, durtolmin, + durtolmax, None if no_progress else printProgress, + "getting time series statistics") + + if no_progress: + sys.stdout.write("done\n") + else: + sys.stdout.write("\n") + + + # *** Compute rankings ************************************************** + + # Step 1: Create a table containing all ranking statistics (one row per + # benchmark/metric): + sys.stdout.write("creating table for all ranking stats ... ") + sys.stdout.flush() + table = getAllRankingStats(bmstats_list) + sys.stdout.write("done\n") + sys.stdout.flush() + + # Step 2: Sort the table individually for each ranking statistic and + # register the ranking positions in the database: + context1_id = getContext(host_id, platform_id, branch_id, snapshots[0][0]) + if context1_id == -1: + print "error: failed to find context for start snapshot" + sys.exit(1) + nameToIndex = { "QS": 3, "LCSSR": 4, "LCSSI": 5, "LCSS1R": 6, "LCSS1I": 7 } + for name in nameToIndex: + sys.stdout.write("registering ranking for " + name + " ... ") + sys.stdout.flush() + registerRanking( + table, nameToIndex[name], name, context1_id, context2_id) + sys.stdout.write("done\n") + sys.stdout.flush() + +# --- END Global functions ---------------------------------------------- + + +# --- BEGIN Main program ---------------------------------------------- + +options, http_get = getOptions() + +if "help" in options: + printVerboseUsage() + sys.exit(0) + +if (not ("db" in options and "host" in options and "platform" in options and + "branch" in options and "sha1" in options)): + printUsage() + sys.exit(0) + +if not isValidSHA1(options["sha1"]): + print "error: invalid SHA-1:", options["sha1"] + sys.exit(1) + +setDatabase( + options["dbhost"] if "dbhost" in options else None, + options["dbport"] if "dbport" in options else None, + options["db"]) + +host_id = textToId("host", options["host"]) +if host_id == -1: + print "error: no such host:", options["host"] + sys.exit(1) +platform_id = textToId("platform", options["platform"]) +if platform_id == -1: + print "error: no such platform:", options["platform"] + sys.exit(1) +branch_id = textToId("branch", options["branch"]) +if branch_id == -1: + print "error: no such branch:", options["branch"] + sys.exit(1) +sha12_id = textToId("sha1", options["sha1"]) +if sha12_id == -1: + print "error: no such SHA-1:", options["sha1"] + sys.exit(1) + +context2_id = getContext(host_id, platform_id, branch_id, sha12_id) +if context2_id == -1: + print "error: no results found for this context" + sys.exit(1) + +updateRankings( + host_id, platform_id, branch_id, sha12_id, context2_id, + ("noprogress" in options) and ( + (options["noprogress"] == "1") + or (options["noprogress"].lower() == "true"))) + +# Write to database: +commit() + +print "rankings computation done" + +# --- END Main program ---------------------------------------------- diff --git a/scripts/finalizeresults.py b/scripts/finalizeresults.py deleted file mode 100755 index dafa858..0000000 --- a/scripts/finalizeresults.py +++ /dev/null @@ -1,353 +0,0 @@ -#!/usr/bin/env python - -import sys -from dbaccess import setDatabase, execQuery, commit -from misc import ( - getOptions, textToId, getAllSnapshots, getLastRankingSnapshot, getContext, - isValidSHA1, getBMTimeSeriesStatsList) - - -# --- BEGIN Global functions ---------------------------------------------- - -def printUsage(): - print ( - "usage:" + sys.argv[0] + - " --help | [--dbhost H --dbport P] --db D --host H --platform P " - "--branch B --sha1 S") - -def printVerboseUsage(): - printUsage() - print "\noptions:" - print( - " --help: This help.") - print( - " --dbhost: The database server host (overriding the default).") - print( - " --dbport: The database server port (overriding the default).") - print( - " --db: The database. One of 'bm' or 'bm-dev' (the latter " - "intended for experimentation).") - print( - " --host: The physical machine on which the results were " - "produced (e.g. barbarella or 172.24.90.79).") - print( - "--platform: The OS/compiler/architecture combination " - "(e.g. linux-g++-32).") - print( - " --branch: The product branch (e.g. 'qt 4.6', 'qt 4.7', or " - "'qt master').") - print( - " --sha1: The tested revision within the branch. Can be " - "extracted using 'git log -1 --pretty=format:%H' (assuming the " - "tested revision is the current head revision).") - - -# ### 2 B DOCUMENTED! -def printProgress(p, lead): - sys.stdout.write(lead + " ... (" + "{0:.2f}".format(p) + " %)\r") - sys.stdout.flush() - - -# ### 2 B DOCUMENTED! -# NOTE: This function is currently duplicated elsewhere in JavaScipt! -def changeMagnitudeScore(change): - max_change = 2.0 - abs_change = (1.0 / change) if change < 1 else change - return (min(abs_change, max_change) - 1.0) / (max_change - 1.0) - - -# ### 2 B DOCUMENTED! -# NOTE: This function is currently duplicated elsewhere in JavaScript! -def qualityScore(lsd, ni, nz, nc, mdrse): - max_bad_snapshots = 30 # experimental; maybe use max durability score? - max_sample_size = 5; - max_LSD = max_bad_snapshots; - max_NI = max_bad_snapshots * max_sample_size; - max_NZ = max_bad_snapshots * max_sample_size; - max_NC = max_bad_snapshots; - - lsd_score = 0 if (lsd == -1) else min(1, lsd / float(max_LSD)); - ni_score = min(1, ni / float(max_NI)); - nz_score = min(1, nz / float(max_NZ)); - nc_score = min(1, nc / float(max_NC)); - mdrse_score = 0 if (mdrse == -1) else (mdrse / 100.0); - - return (lsd_score + ni_score + nz_score + nc_score + mdrse_score) / 5.0; - - -# Registers the ranking for a given statistic. context1_id and context2_id -# refer to the first and last snapshot respectively in the interval used for -# computing the rankings. -# Assumptions: -# - A high value should be ranked above a small one. -# - A negative value is undefined and gets an invalid ranking position, i.e. -1. -def registerRanking(table, stat_index, stat_name, context1_id, context2_id): - - table.sort(key=lambda x: x[stat_index], reverse=True) - - stat_id = textToId("rankingStat", stat_name) - assert stat_id >= 0 - - row_pos = 0 - ranking_pos = 0 - for row in table: - benchmark_id = row[0] - metric_id = row[1] - lc_timestamp = row[2] - stat_value = row[stat_index] - - # The following statement ensures the following conditions: - # - A negative value gets an invalid ranking position, i.e. -1 - # - Equal values get the same ranking position. - # - The ranking position of benchmark B indicates the number of - # benchmarks ranked higher than B (i.e. having a smaller ranking - # position). - if stat_value < 0: - ranking_pos = -1 - # Note that the remaining values will now be negative, so updating - # row_pos and prev_stat_value is no longer necessary! - else: - if (row_pos > 0) and (stat_value != prev_stat_value): - ranking_pos = row_pos - row_pos = row_pos + 1 - prev_stat_value = stat_value - - # Insert or update the corresponding row in the 'ranking' table: - query = ( - "SELECT merge_ranking(" - + str(context1_id) - + ", " + str(context2_id) - + ", " + str(benchmark_id) - + ", " + str(metric_id) - + ", " + str(lc_timestamp) - + ", " + str(stat_id) - + ", " + str(stat_value) - + ", " + str(ranking_pos) - + ");" - ) - execQuery(query, False) - - -# ### 2 B DOCUMENTED! -def getAllRankingStats(bmstats_list): - table = [] - for stats in bmstats_list: - - # NOTE: - # - All of the ranking statistics are of type "higher is better" - # (a high value is ranked better than a low value). - # - Moreover, all present/defined values are non-negative. - # - This means that representing absent/undefined values as -1 is ok, - # since this ensures lowest ranking. - - benchmark_id = stats["benchmark_id"] - metric_id = stats["metric_id"] - lc_timestamp = stats["lc_timestamp"] - lsd = stats["lsd"] - ni = stats["ni"] - nz = stats["nz"] - nc = stats["nc"] - mdrse = stats["med_of_rses"] - rsemd = stats["rse_of_meds"] - - qs = qualityScore(lsd, ni, nz, nc, mdrse) - - lc = stats["lc"] - if lc >= 0.0: - lcgss = stats["lc_gsep_score"] - lclss = stats["lc_lsep_score"] - lcds1 = stats["lc_dur1_score"] - lcds2 = stats["lc_dur2_score"] - lcms = changeMagnitudeScore(lc) - lcss1 = (lcms + lcgss + lclss + lcds1) / 4.0 - lcss = (lcms + lcgss + lclss + lcds1 + lcds2) / 5.0 - if lc < 1.0: - lcssr = lcss - lcss1r = lcss1 - lcssi = lcss1i = -1 - else: - lcssi = lcss - lcss1i = lcss1 - lcssr = lcss1r = -1 - else: - lcssr = lcssi = lcss1r = lcss1i = -1 - - table.append( - (benchmark_id, metric_id, lc_timestamp, qs, lcssr, lcssi, lcss1r, - lcss1i)) - - return table - - -# ### 2 B DOCUMENTED! -def getFirstUploadTimestamp(snapshots, sha1_id): - try: - return snapshots[zip(*snapshots)[0].index(sha1_id)][1] - except ValueError: - return -1 - - -# ### 2 B DOCUMENTED! -def updateRankings(host_id, platform_id, branch_id, sha12_id, context2_id): - - # Get all snapshots matching the host/platform/branch combination: - sys.stdout.write("getting snapshots ... ") - sys.stdout.flush() - snapshots = getAllSnapshots(host_id, platform_id, branch_id) - sys.stdout.write("done\n") - sys.stdout.flush() - - - # Rankings will normally be computed once a day for each - # host/platform/branch combination (note the tradeoff between update - # frequency and database size): - ranking_interval = 3600 * 24 # secs in a day - - # Rankings will be updated if at least one of the following - # conditions eventually becomes True: - force_cond = empty_cond = interval_cond = False - - force_ranking = False - #force_ranking = True # Uncomment for testing - - force_cond = force_ranking - - if not force_cond: - last_ranking_sha1_id, last_ranking_timestamp = getLastRankingSnapshot( - host_id, platform_id, branch_id) - empty_cond = last_ranking_sha1_id < 0 - if not empty_cond: - assert last_ranking_timestamp >= 0 - - target_timestamp = getFirstUploadTimestamp(snapshots, sha12_id) - if target_timestamp < 0: - print ( - "error: failed to extract target_timestamp " - "(error in command-line args?)") - sys.exit(1) - - interval_cond = ( - (target_timestamp - last_ranking_timestamp) > ranking_interval) - - if not (force_cond or empty_cond or interval_cond): - print ( - "not updating rankings ('force', 'empty', and 'interval' " - "conditions all failed)") - return - - print ( - "updating rankings ('force' cond.: " + str(force_cond) + - "; 'empty' cond.: " + str(empty_cond) + - "; 'interval' cond.: " + str(interval_cond) + ") ...") - - # For simplicity we hardcode the tolerances for now: - difftol = 1.1 - durtolmin = 3 - durtolmax = 30 - - # Determine the target snapshot range: - # (The range should end at the snapshot given on the command-line and begin - # at the snapshot that is 2 * durtolmax snapshots back in time, or, if no - # such snapshot exists, the first available snapshot.) - try: - sha12_pos = zip(*snapshots)[0].index(sha12_id) - except ValueError: - print "no observations found for SHA-1 ID:", sha12_id - sys.exit(1) - sha11_pos = max(0, (sha12_pos - 2 * durtolmax) + 1) - snapshots = snapshots[sha11_pos:(sha12_pos + 1)] - if len(snapshots) < 2: - print ( - "no observations found before SHA-1 ID: " + sha12_id + - " (computing rankings makes no sense)") - sys.exit(1) - - # Get time series statistics for all benchmarks: - bmstats_list = getBMTimeSeriesStatsList( - host_id, platform_id, branch_id, snapshots, None, difftol, durtolmin, - durtolmax, printProgress, "getting time series statistics") - - sys.stdout.write("\n") - - - # *** Compute rankings ************************************************** - - # Step 1: Create a table containing all ranking statistics (one row per - # benchmark/metric): - sys.stdout.write("creating table for all ranking stats ... ") - sys.stdout.flush() - table = getAllRankingStats(bmstats_list) - sys.stdout.write("done\n") - sys.stdout.flush() - - # Step 2: Sort the table individually for each ranking statistic and - # register the ranking positions in the database: - context1_id = getContext(host_id, platform_id, branch_id, snapshots[0][0]) - if context1_id == -1: - print "error: failed to find context for start snapshot" - sys.exit(1) - nameToIndex = { "QS": 3, "LCSSR": 4, "LCSSI": 5, "LCSS1R": 6, "LCSS1I": 7 } - for name in nameToIndex: - sys.stdout.write("registering ranking for " + name + " ...\r") - sys.stdout.flush() - registerRanking( - table, nameToIndex[name], name, context1_id, context2_id) - - sys.stdout.write("\n") - -# --- END Global functions ---------------------------------------------- - - -# --- BEGIN Main program ---------------------------------------------- - -options, http_get = getOptions() - -if "help" in options: - printVerboseUsage() - sys.exit(0) - -if (not ("db" in options and "host" in options and "platform" in options and - "branch" in options and "sha1" in options)): - printUsage() - sys.exit(0) - -if not isValidSHA1(options["sha1"]): - print "error: invalid SHA-1:", options["sha1"] - sys.exit(1) - -setDatabase( - options["dbhost"] if "dbhost" in options else None, - options["dbport"] if "dbport" in options else None, - options["db"]) - -host_id = textToId("host", options["host"]) -if host_id == -1: - print "error: no such host:", options["host"] - sys.exit(1) -platform_id = textToId("platform", options["platform"]) -if platform_id == -1: - print "error: no such platform:", options["platform"] - sys.exit(1) -branch_id = textToId("branch", options["branch"]) -if branch_id == -1: - print "error: no such branch:", options["branch"] - sys.exit(1) -sha12_id = textToId("sha1", options["sha1"]) -if sha12_id == -1: - print "error: no such SHA-1:", options["sha1"] - sys.exit(1) - -context2_id = getContext(host_id, platform_id, branch_id, sha12_id) -if context2_id == -1: - print "error: no results found for this context" - sys.exit(1) - - -updateRankings(host_id, platform_id, branch_id, sha12_id, context2_id) - -# Write to database: -commit() - -print "finalization done" - -# --- END Main program ---------------------------------------------- diff --git a/scripts/uploadresults.py b/scripts/uploadresults.py index 3c92691..f0db5a7 100755 --- a/scripts/uploadresults.py +++ b/scripts/uploadresults.py @@ -1,9 +1,11 @@ #!/usr/bin/env python import sys +from subprocess import Popen, PIPE from xml.dom.minidom import parse from dbaccess import setDatabase, execQuery, commit -from misc import getOptions, textToId, isValidSHA1, getContext +from misc import ( + getOptions, textToId, idToText, isValidSHA1, getContext, getAllSnapshots) # --- BEGIN Global functions ---------------------------------------------- @@ -253,6 +255,75 @@ def rankingsExist(host, platform, branch, sha1): return len(matches) > 0 +# Returns the context ID if found, otherwise -1: +def getContextIdFromNames(options): + host_id = textToId("host", options["host"]) + platform_id = textToId("platform", options["platform"]) + branch_id = textToId("branch", options["branch"]) + sha1_id = textToId("sha1", options["sha1"]) + return getContext(host_id, platform_id, branch_id, sha1_id) + + +# Returns True iff this context exists: +def contextExists(options): + return getContextIdFromNames(options) != -1 + + +# Returns True iff no more results are to be expected for this context: +def contextComplete(options): + + max_sample_size = 5 # WARNING: This value must match the corresponding value + # in the script that triggers benchmark execution. + + context_id = getContextIdFromNames(options) + sample_size = execQuery( + "SELECT count(*) FROM" + " (SELECT DISTINCT uploadId from result where contextId=%d) AS foo;" + % context_id)[0][0] + + return sample_size >= max_sample_size + + +# Executes the external computerankings.py script with appropriate arguments. +def execComputeRankings(options, use_latest_snapshot): + + if use_latest_snapshot: + # Attempt to use the latest available snapshot for this + # host/platform/branch combination as the actual snapshot: + host_id = textToId("host", options["host"]) + platform_id = textToId("platform", options["platform"]) + branch_id = textToId("branch", options["branch"]) + snapshots = getAllSnapshots(host_id, platform_id, branch_id) + if len(snapshots) > 0: + actual_snapshot = idToText("sha1", snapshots[-1][0]) + else: + return # special case when no snapshots exist yet + else: + actual_snapshot = options["sha1"] + + cmd = [ + "computerankings.py", "--db", options["db"], "--host", options["host"], + "--platform", options["platform"], "--branch", options["branch"], + "--sha1", actual_snapshot, "--noprogress", "true"] + if "dbhost" in options: + cmd += ["--dbhost", options["dbhost"]] + if "dbport" in options: + cmd += ["--dbport", options["dbport"]] + + p = Popen(cmd, stdout = PIPE, stderr = PIPE) + stdout, stderr = p.communicate() + if (p.returncode != 0): + print "failed to execute command '" + cmd + "':" + print " return code:", p.returncode + print " stdout: >%s<" % stdout.strip() + print " stderr: >%s<" % stderr.strip() + else: + print "computerankings.py executed successfully:" + print " return code:", p.returncode + print " stdout: >%s<" % stdout.strip() + print " stderr: >%s<" % stderr.strip() + + # --- END Global functions ---------------------------------------------- @@ -281,27 +352,37 @@ setDatabase( # Reject uploading if rankings exist for this context, since that would # require a recomputation of these rankings. # -# (Note that this check if sufficient as long as we assume that results -# are uploaded in an order that is consistent with the order of the -# corresponding SHA-1s in the branch in question (i.e. that results will -# never be uploaded for a SHA-1 that precedes (is "older" than) a SHA-1 for -# which results already exist). This assumption ensures that results for a -# new SHA-1 will not alter the input for any existing ranking.) +# (Note that we only need to check the current SHA-1 as long as we assume that +# results are uploaded in an order that is consistent with the order of the +# corresponding SHA-1s in the branch in question: Rankings for earlier SHA-1s +# will not be affected by results for this SHA-1, and rankings for later +# SHA-1s cannot exist a this point (by assumption).) if rankingsExist( options["host"], options["platform"], options["branch"], options["sha1"]): print "error: this context has already been finalized" sys.exit(1) +# If this is the first set of results for the current context, we finalize the +# results for the same context, only one snapshot back: +if not contextExists(options): + execComputeRankings(options, True) + +# Parse results and store them in the database: try: results = extractResults(options["file"]) except BaseException as e: print "error: failed to parse XML file:", e.args[0] sys.exit(1) - uploadToDatabase( options["host"], options["platform"], options["branch"], options["sha1"], results) print "uploading done" +# If no more results are expected in this context, we can finalize the +# results already at this point: +if contextComplete(options): + execComputeRankings(options, False) + + # --- END Main program ---------------------------------------------- -- cgit v1.2.3