diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/computerankings.py | 359 | ||||
| -rw-r--r-- | scripts/getnamemappings.py | 34 | ||||
| -rw-r--r-- | scripts/getrankings.py | 219 | ||||
| -rwxr-xr-x | scripts/getstats.py | 80 | ||||
| -rw-r--r-- | scripts/gettestcaseswithchanges.py | 30 | ||||
| -rw-r--r-- | scripts/gettopchanges.py | 181 | ||||
| -rw-r--r-- | scripts/listcontexts.py | 46 | ||||
| -rw-r--r-- | scripts/misc.py | 429 | ||||
| -rwxr-xr-x | scripts/updateallchanges.py | 96 | ||||
| -rwxr-xr-x | scripts/updatechanges.py | 201 | ||||
| -rwxr-xr-x | scripts/uploadresults.py | 130 |
11 files changed, 990 insertions, 815 deletions
diff --git a/scripts/computerankings.py b/scripts/computerankings.py deleted file mode 100755 index 8ad1fd3..0000000 --- a/scripts/computerankings.py +++ /dev/null @@ -1,359 +0,0 @@ -#!/usr/bin/env python - -import sys -from dbaccess import setDatabase, execQuery, commit -from misc import ( - getOptions, textToId, getAllSnapshots, getLastRankingSnapshot, getContext, - isValidSHA1, getBMTimeSeriesStatsList) - - -# --- BEGIN Global functions ---------------------------------------------- - -def printUsage(): - sys.stderr.write( - "usage: " + sys.argv[0] + - " --help | [--dbhost H] [--dbport P] --db D --host H --platform P " - "--branch B --sha1 S [--noprogress NP]\n") - -def printVerboseUsage(): - printUsage() - sys.stderr.write("\noptions:\n") - sys.stderr.write( - " --help: This help.\n") - sys.stderr.write( - " --dbhost: The database server host (overriding the default).\n") - sys.stderr.write( - " --dbport: The database server port (overriding the default).\n") - sys.stderr.write( - " --db: The database. One of 'bm' or 'bm-dev' (the latter " - "intended for experimentation).\n") - sys.stderr.write( - " --host: The physical machine on which the results were " - "produced (e.g. barbarella or 172.24.90.79).\n") - sys.stderr.write( - "--platform: The OS/compiler/architecture combination " - "(e.g. linux-g++-32).\n") - sys.stderr.write( - " --branch: The product branch (e.g. 'qt 4.6', 'qt 4.7', or " - "'qt master').\n") - sys.stderr.write( - " --sha1: The tested revision within the branch. Can be " - "extracted using 'git log -1 --pretty=format:%H' (assuming the " - "tested revision is the current head revision).\n") - sys.stderr.write( - " --noprogress: Specify \'true\' to disable progress indicator.\n") - - -# ### 2 B DOCUMENTED! -def printProgress(p, lead): - sys.stdout.write(lead + " ... (" + "{0:.2f}".format(p) + " %)\r") - sys.stdout.flush() - - -# ### 2 B DOCUMENTED! -# NOTE: This function is currently duplicated elsewhere in JavaScipt! -def changeMagnitudeScore(change): - max_change = 2.0 - abs_change = (1.0 / change) if change < 1 else change - return (min(abs_change, max_change) - 1.0) / (max_change - 1.0) - - -# ### 2 B DOCUMENTED! -# NOTE: This function is currently duplicated elsewhere in JavaScript! -def qualityScore(lsd, ni, nz, nc, mdrse): - max_bad_snapshots = 10 # experimental; maybe use max durability score? - max_sample_size = 5; - max_LSD = max_bad_snapshots; - max_NI = max_bad_snapshots * max_sample_size; - max_NZ = max_bad_snapshots * max_sample_size; - max_NC = max_bad_snapshots; - - lsd_score = 0 if (lsd == -1) else min(1, lsd / float(max_LSD)); - ni_score = min(1, ni / float(max_NI)); - nz_score = min(1, nz / float(max_NZ)); - nc_score = min(1, nc / float(max_NC)); - mdrse_score = 0 if (mdrse == -1) else (mdrse / 100.0); - - return (lsd_score + ni_score + nz_score + nc_score + mdrse_score) / 5.0; - - -# Registers the ranking for a given statistic. context1_id and context2_id -# refer to the first and last snapshot respectively in the interval used for -# computing the rankings. -# Assumptions: -# - A high value should be ranked above a small one. -# - A negative value is undefined and gets an invalid ranking position, i.e. -1. -def registerRanking(table, stat_index, stat_name, context1_id, context2_id): - - table.sort(key=lambda x: x[stat_index], reverse=True) - - stat_id = textToId("rankingStat", stat_name) - assert stat_id >= 0 - - row_pos = 0 - ranking_pos = 0 - for row in table: - benchmark_id = row[0] - metric_id = row[1] - lc_timestamp = row[2] - stat_value = row[stat_index] - - # The following statement ensures the following conditions: - # - A negative value gets an invalid ranking position, i.e. -1 - # - Equal values get the same ranking position. - # - The ranking position of benchmark B indicates the number of - # benchmarks ranked higher than B (i.e. having a smaller ranking - # position). - if stat_value < 0: - ranking_pos = -1 - # Note that the remaining values will now be negative, so updating - # row_pos and prev_stat_value is no longer necessary! - else: - if (row_pos > 0) and (stat_value != prev_stat_value): - ranking_pos = row_pos - row_pos = row_pos + 1 - prev_stat_value = stat_value - - # Insert or update the corresponding row in the 'ranking' table: - execQuery( - "SELECT merge_ranking(%s, %s, %s, %s, %s, %s, %s, %s)", - (context1_id, context2_id, benchmark_id, metric_id, - lc_timestamp, stat_id, stat_value, ranking_pos), - False) - - -# ### 2 B DOCUMENTED! -def getAllRankingStats(bmstats_list): - table = [] - for stats in bmstats_list: - - # NOTE: - # - All of the ranking statistics are of type "higher is better" - # (a high value is ranked better than a low value). - # - Moreover, all present/defined values are non-negative. - # - This means that representing absent/undefined values as -1 is ok, - # since this ensures lowest ranking. - - benchmark_id = stats["benchmark_id"] - metric_id = stats["metric_id"] - lc_timestamp = stats["lc_timestamp"] - lsd = stats["lsd"] - ni = stats["ni"] - nz = stats["nz"] - nc = stats["nc"] - mdrse = stats["med_of_rses"] - rsemd = stats["rse_of_meds"] - - qs = qualityScore(lsd, ni, nz, nc, mdrse) - - lc = stats["lc"] - if lc >= 0.0: - lcgss = stats["lc_gsep_score"] - lclss = stats["lc_lsep_score"] - lcds1 = stats["lc_dur1_score"] - lcds2 = stats["lc_dur2_score"] - lcms = changeMagnitudeScore(lc) - lcss1 = lcms * lcgss * lclss * lcds1 - lcss = lcss1 * lcds2 - if lc < 1.0: - lcssr = lcss - lcss1r = lcss1 - lcssi = lcss1i = -1 - else: - lcssi = lcss - lcss1i = lcss1 - lcssr = lcss1r = -1 - else: - lcssr = lcssi = lcss1r = lcss1i = -1 - - table.append( - (benchmark_id, metric_id, lc_timestamp, qs, lcssr, lcssi, lcss1r, - lcss1i)) - - return table - - -# ### 2 B DOCUMENTED! -def getFirstUploadTimestamp(snapshots, sha1_id): - try: - return snapshots[zip(*snapshots)[0].index(sha1_id)][1] - except ValueError: - return -1 - - -# ### 2 B DOCUMENTED! -def updateRankings( - host_id, platform_id, branch_id, sha12_id, context2_id, no_progress): - - # Get all snapshots matching the host/platform/branch combination: - sys.stdout.write("getting snapshots ... ") - sys.stdout.flush() - snapshots = getAllSnapshots(host_id, platform_id, branch_id) - sys.stdout.write("done\n") - sys.stdout.flush() - - - # Rankings will normally be computed once a day for each - # host/platform/branch combination (note the tradeoff between update - # frequency and database size): - ranking_interval = 3600 * 24 # secs in a day - - # Rankings will be updated if at least one of the following - # conditions eventually becomes True: - force_cond = empty_cond = interval_cond = False - - force_ranking = False - #force_ranking = True # Uncomment for testing - - force_cond = force_ranking - - if not force_cond: - last_ranking_sha1_id, last_ranking_timestamp = getLastRankingSnapshot( - host_id, platform_id, branch_id) - empty_cond = last_ranking_sha1_id < 0 - if not empty_cond: - assert last_ranking_timestamp >= 0 - - target_timestamp = getFirstUploadTimestamp(snapshots, sha12_id) - if target_timestamp < 0: - sys.stderr.write( - "error: failed to extract target_timestamp " - "(error in command-line args?)\n") - sys.exit(1) - - interval_cond = ( - (target_timestamp - last_ranking_timestamp) > ranking_interval) - - if not (force_cond or empty_cond or interval_cond): - sys.stdout.write( - "not updating rankings ('force', 'empty', and 'interval' " - "conditions all failed)\n") - return - - sys.stdout.write( - "updating rankings ('force' cond.: " + str(force_cond) + - "; 'empty' cond.: " + str(empty_cond) + - "; 'interval' cond.: " + str(interval_cond) + ") ...\n") - - # For simplicity we hardcode the tolerances for now: - difftol = 1.1 - durtolmin = 3 - durtolmax = 10 - - # Determine the target snapshot range: - # (The range should end at the snapshot given on the command-line and begin - # at the snapshot that is 2 * durtolmax snapshots back in time, or, if no - # such snapshot exists, the first available snapshot.) - try: - sha12_pos = zip(*snapshots)[0].index(sha12_id) - except ValueError: - sys.stderr.write( - "no observations found for SHA-1 ID: " + str(sha12_id) + "\n") - sys.exit(1) - sha11_pos = max(0, (sha12_pos - 2 * durtolmax) + 1) - snapshots = snapshots[sha11_pos:(sha12_pos + 1)] - if len(snapshots) < 2: - sys.stderr.write( - "no observations found before SHA-1 ID: " + str(sha12_id) + - " (computing rankings makes no sense)\n") - sys.exit(1) - - # Get time series statistics for all benchmarks: - if no_progress: - sys.stdout.write("getting time series statistics ... ") - bmstats_list = getBMTimeSeriesStatsList( - host_id, platform_id, branch_id, snapshots, None, difftol, durtolmin, - durtolmax, None if no_progress else printProgress, - "getting time series statistics") - - if no_progress: - sys.stdout.write("done\n") - else: - sys.stdout.write("\n") - - - # *** Compute rankings ************************************************** - - # Step 1: Create a table containing all ranking statistics (one row per - # benchmark/metric): - sys.stdout.write("creating table for all ranking stats ... ") - sys.stdout.flush() - table = getAllRankingStats(bmstats_list) - sys.stdout.write("done\n") - sys.stdout.flush() - - # Step 2: Sort the table individually for each ranking statistic and - # register the ranking positions in the database: - context1_id = getContext(host_id, platform_id, branch_id, snapshots[0][0]) - if context1_id == -1: - sys.stderr.write("error: failed to find context for start snapshot\n") - sys.exit(1) - nameToIndex = { "QS": 3, "LCSSR": 4, "LCSSI": 5, "LCSS1R": 6, "LCSS1I": 7 } - for name in nameToIndex: - sys.stdout.write("registering ranking for " + name + " ... ") - sys.stdout.flush() - registerRanking( - table, nameToIndex[name], name, context1_id, context2_id) - sys.stdout.write("done\n") - sys.stdout.flush() - -# --- END Global functions ---------------------------------------------- - - -# --- BEGIN Main program ---------------------------------------------- - -options, http_get = getOptions() - -if "help" in options: - printVerboseUsage() - sys.exit(1) - -if (not ("db" in options and "host" in options and "platform" in options and - "branch" in options and "sha1" in options)): - printUsage() - sys.exit(1) - -if not isValidSHA1(options["sha1"]): - sys.stderr.write("error: invalid SHA-1: " + options["sha1"] + "\n") - sys.exit(1) - -setDatabase( - options["dbhost"] if "dbhost" in options else None, - options["dbport"] if "dbport" in options else None, - options["db"]) - -host_id = textToId("host", options["host"]) -if host_id == -1: - sys.stderr.write("error: no such host: " + options["host"] + "\n") - sys.exit(1) -platform_id = textToId("platform", options["platform"]) -if platform_id == -1: - sys.stderr.write("error: no such platform: " + options["platform"] + "\n") - sys.exit(1) -branch_id = textToId("branch", options["branch"]) -if branch_id == -1: - sys.stderr.write("error: no such branch:" + options["branch"] + "\n") - sys.exit(1) -sha12_id = textToId("sha1", options["sha1"]) -if sha12_id == -1: - sys.stderr.write("error: no such SHA-1:" + options["sha1"] + "\n") - sys.exit(1) - -context2_id = getContext(host_id, platform_id, branch_id, sha12_id) -if context2_id == -1: - sys.stderr.write("error: no results found for this context\n") - sys.exit(1) - -updateRankings( - host_id, platform_id, branch_id, sha12_id, context2_id, - ("noprogress" in options) and ( - (options["noprogress"] == "1") - or (options["noprogress"].lower() == "true"))) - -# Write to database: -commit() - -sys.stdout.write("rankings computation done\n") -sys.exit(0) - -# --- END Main program ---------------------------------------------- diff --git a/scripts/getnamemappings.py b/scripts/getnamemappings.py new file mode 100644 index 0000000..2768449 --- /dev/null +++ b/scripts/getnamemappings.py @@ -0,0 +1,34 @@ +import sys, json +from dbaccess import execQuery +from misc import printJSONHeader + + +class GetNameMappings: + + def __init__(self): + pass + + def execute(self): + self.hosts = dict(execQuery("SELECT id, value FROM host", ())) + self.platforms = dict(execQuery("SELECT id, value FROM platform", ())) + self.branches = dict(execQuery("SELECT id, value FROM branch", ())) + self.sha1s = dict(execQuery("SELECT id, value FROM sha1", ())) + self.benchmarks = dict(execQuery("SELECT id, value FROM benchmark", ())) + self.metrics = dict(execQuery("SELECT id, value FROM metric", ())) + self.writeOutput() + + def writeOutputAsJSON(self): + printJSONHeader() + json.dump({ + 'hosts': self.hosts, + 'platforms': self.platforms, + 'branches': self.branches, + 'sha1s': self.sha1s, + 'benchmarks': self.benchmarks, + 'metrics': self.metrics + }, sys.stdout) + + +class GetNameMappingsAsJSON(GetNameMappings): + def writeOutput(self): + self.writeOutputAsJSON() diff --git a/scripts/getrankings.py b/scripts/getrankings.py deleted file mode 100644 index 45cc9a3..0000000 --- a/scripts/getrankings.py +++ /dev/null @@ -1,219 +0,0 @@ -import sys -import json -from dbaccess import execQuery, database -from misc import ( - textToId, idToText, getContext, getTimestampFromContext, getSnapshots, - getRankingContexts, benchmarkToComponents, printJSONHeader) - -class GetRankings: - - def __init__( - self, host, platform, branch, sha12, test_case_filter, maxsize): - self.host = host - self.host_id = textToId('host', self.host) - self.platform = platform - self.platform_id = textToId('platform', self.platform) - self.branch = branch - self.branch_id = textToId('branch', self.branch) - self.context2_id = getContext( - self.host_id, self.platform_id, self.branch_id, - textToId('sha1', sha12)) - self.test_case_filter = test_case_filter - self.maxsize = maxsize - - - # Returns -1, 0, and 1 if ranking position x is considered less than, - # equal to, and greater than ranking position y respectively. - # Note: a negative ranking position is considered worse (i.e. effectively - # treated as having an "infinite" ranking position) than any non-negative - # ranking position. - def cmp_rank_pos(self, x, y): - if x < 0: - return 1 - elif y < 0: - return -1 - elif x < y: - return -1 - elif x > y: - return 1 - else: - return 0 - - - # Gets all rankings matching the context/metric combination combination. - def getRankings(self): - - if self.context2_id < 0: - print "error: invalid context" - sys.exit(1) - - # Find the previous context (if any) for which rankings exist: - ranking_contexts = getRankingContexts( - self.host_id, self.platform_id, self.branch_id) - curr_index = zip(*ranking_contexts)[0].index(self.context2_id) - if curr_index < (len(ranking_contexts) - 1): - context2_prev_id = ranking_contexts[curr_index + 1][0] - else: - context2_prev_id = -1 # No rankings before this context - - rankings = {} - context_ids = set([self.context2_id]) # Affected context IDs - - - # Get all time series notes: - qres = execQuery( - "SELECT benchmarkId, metricId, note FROM timeSeriesAnnotation" - " WHERE hostId = %s AND platformId = %s AND branchId = %s", - (self.host_id, self.platform_id, self.branch_id)) - notes = {} - for benchmark_id, metric_id, note in qres: - notes[benchmark_id, metric_id] = note - - - # Get rankings for each statistic: - stat_infos = execQuery("SELECT id, value FROM rankingStat", ()) - for stat_id, stat_name in stat_infos: - - # Get the unsorted ranking information: - ranking_all = execQuery( - "SELECT benchmarkId, metricId, context1Id, pos, value," - " lastChangeTimestamp" - " FROM ranking" - " WHERE context2Id = %s" - " AND statId = %s", - (self.context2_id, stat_id)) - - ranking = [] - - # Apply test case filter and add notes: - for row in ranking_all: - benchmark_id = row[0] - benchmark = idToText("benchmark", benchmark_id) - test_case, test_function, data_tag = ( - benchmarkToComponents(benchmark)) - if ((self.test_case_filter == None) - or (test_case in self.test_case_filter)): - - # Append note if any: - metric_id = row[1] - try: - note = notes[benchmark_id, metric_id] - except KeyError: - note = "" - - ranking.append(( - benchmark_id, metric_id, row[2], row[3], row[4], - row[5], note)) - - - for row in ranking: - context_ids.add(row[2]) - - # Sort the table in ascending order on the 'pos' column, but - # so that negative positions are ranked below any other positions: - ranking.sort(key=lambda row: row[3], cmp=self.cmp_rank_pos) - - # Keep only the 'maxsize' highest ranked benchmarks: - ranking = ranking if (self.maxsize < 0) else ranking[:self.maxsize] - - if context2_prev_id >= 0: - # Compute deltas from previous ranking: - ranking_prev_list = execQuery( - "SELECT benchmarkId, metricId, pos" - " FROM ranking" - " WHERE context2Id = %s" - " AND statId = %s", - (context2_prev_id, stat_id)) - ranking_prev = {} - for benchmark_id, metric_id, pos in ranking_prev_list: - ranking_prev[benchmark_id, metric_id] = pos - - # Append deltas where applicable: - ranking_without_deltas = ranking - ranking = [] - - for (benchmark_id, metric_id, context1_id, pos, value, - lc_timestamp, note) in ranking_without_deltas: - row = [benchmark_id, metric_id, context1_id, pos, value, - lc_timestamp, note] - if pos >= 0: - try: - pos_prev = ranking_prev[benchmark_id, metric_id] - if pos_prev >= 0: - delta = pos_prev - pos - row.append(delta) - except KeyError: - pass - ranking.append(row) - - - # Add to main list: - rankings[stat_name.lower()] = ranking; - - - # Extract affected SHA-1s: - assert len(context_ids) > 0 - sha1_infos = execQuery( - "SELECT context.id, sha1Id, sha1.value" - " FROM context, sha1" - " WHERE context.id IN" - " (%s" + ", %s"*(len(context_ids) - 1) + ")" + - " AND sha1Id = sha1.id", - tuple(context_ids)) - - - return sha1_infos, rankings - - - # Extracts the individual snapshots in the maximum range spanned by - # the SHA-1s in sha1_infos: - def getSnapshotsInMaxRange(self, sha1_infos): - - min_timestamp = max_timestamp = first_sha1_id = last_sha1_id = None - for context_id, sha1_id, sha1 in sha1_infos: - timestamp = getTimestampFromContext(context_id) - if min_timestamp == None: - min_timestamp = max_timestamp = timestamp - first_sha1_id = last_sha1_id = sha1_id - elif timestamp < min_timestamp: - min_timestamp = timestamp - first_sha1_id = sha1_id - elif timestamp > max_timestamp: - max_timestamp = timestamp - last_sha1_id = sha1_id - - snapshots = getSnapshots( - self.host_id, self.platform_id, self.branch_id, first_sha1_id, - last_sha1_id) - - return snapshots - - - def execute(self): - self.sha1_infos, self.rankings = self.getRankings() - self.snapshots = self.getSnapshotsInMaxRange(self.sha1_infos) - - self.benchmarks = execQuery("SELECT id, value FROM benchmark", ()) - self.metrics = execQuery("SELECT id, value FROM metric", ()) - - self.writeOutput() - - - def writeOutputAsJSON(self): - printJSONHeader() - json.dump({ - 'database': database(), - 'host': self.host, - 'platform': self.platform, - 'branch': self.branch, - 'benchmarks': self.benchmarks, - 'metrics': self.metrics, - 'snapshots': map( - lambda s: (idToText("sha1", s[0]), s[1]), self.snapshots), - 'rankings': self.rankings - }, sys.stdout) - - -class GetRankingsAsJSON(GetRankings): - def writeOutput(self): - self.writeOutputAsJSON() diff --git a/scripts/getstats.py b/scripts/getstats.py index d0b7d59..735b28a 100755 --- a/scripts/getstats.py +++ b/scripts/getstats.py @@ -9,8 +9,10 @@ from getresultdetails2 import GetResultDetails2AsJSON from gettimeseriesstats import GetTimeSeriesStatsAsJSON from gettimeseriesdetails import GetTimeSeriesDetailsAsJSON from getsnapshots import GetSnapshotsAsJSON -from getrankings import GetRankingsAsJSON from settimeseriesnote import SetTimeSeriesNote +from gettopchanges import GetTopChangesAsJSON +from getnamemappings import GetNameMappingsAsJSON +from gettestcaseswithchanges import GetTestCasesWithChangesAsJSON from dbaccess import setDatabase from misc import getOptions, printErrorAsJSON @@ -18,13 +20,25 @@ import sys # --- BEGIN Global functions ---------------------------------------------- +# Returns true iff name exists in options and is true. +def boolOption(options, name): + if name in options: + try: + res = (int(options[name]) != 0) + except: + res = (options[name].lower() == "true") + else: + res = False + return res + + # Returns a command instance. def createCommand(options, http_get): def printUsageError(): error = ( "usage: " + sys.argv[0] + " [--dbhost H --dbport P] --db D + \\\n" - " --cmd contexts [--rankedonly R] | \\\n" + " --cmd contexts | \\\n" " --cmd testcases1 --host H --platform P --branch B " "--sha1 S | \\\n" " --cmd testcases2 --host1 H --platform1 P --branch1 B " @@ -45,11 +59,13 @@ def createCommand(options, http_get): "--durtolmax T --benchmark BM --metric M | \\\n" " --cmd snapshots --host H --platform P " "--branch B --sha11 S --sha12 S | \\\n" - " --cmd rankings --host H --platform P " - "--branch B --sha1 S [--testcasefilter 'TC1 TC2 ...'] " "[--maxsize M] | \\\n" " --cmd settimeseriesnote --host H --platform P " - "--branch B --benchmark B --metric M --note N") + "--branch B --benchmark B --metric M --note N | \\\n" + " --cmd topchanges --regressions R --last L --timescope T " + "--premature P --limit L [--testcasefilter 'TC1 TC2 ...'] | \\\n" + " --cmd namemappings | \\\n" + " --cmd testcaseswithchanges") if http_get: printErrorAsJSON("usage error") @@ -82,15 +98,7 @@ def createCommand(options, http_get): # --- 'contexts' --------------------------------- if cmd == "contexts": - if "rankedonly" in options: - try: - ranked_only = (int(options["rankedonly"]) != 0) - except: - ranked_only = (options["rankedonly"].lower() == "true") - else: - ranked_only = False - - return ListContextsAsJSON(ranked_only) + return ListContextsAsJSON() # --- 'testcases1' --------------------------------- elif cmd == "testcases1": @@ -252,26 +260,6 @@ def createCommand(options, http_get): return GetSnapshotsAsJSON(host, platform, branch, sha11, sha12) - # --- 'rankings' --------------------------------- - elif cmd == "rankings": - if ("host" in options and "platform" in options and - "branch" in options and "sha1" in options): - host = options["host"] - platform = options["platform"] - branch = options["branch"] - sha1 = options["sha1"] - - if "maxsize" in options: - try: - maxsize = int(options["maxsize"]) - except: - raise BaseException("'maxsize' not an integer") - else: - maxsize = 10 - - return GetRankingsAsJSON( - host, platform, branch, sha1, test_case_filter, maxsize) - # --- 'settimeseriesnote' --------------------------------- # ### Hm ... this command doesn't really get statistics, so maybe # rename getstats.py to something more generic @@ -290,6 +278,30 @@ def createCommand(options, http_get): return SetTimeSeriesNote( host, platform, branch, benchmark, metric, note) + # --- 'topchanges' --------------------------------- + elif cmd == "topchanges": + if ("regressions" in options and "last" in options and + "timescope" in options and "premature" in options and + "limit" in options): + regressions = boolOption(options, "regressions") + last = boolOption(options, "last") + timescope = int(options["timescope"]) + premature = boolOption(options, "premature") + limit = int(options["limit"]) + + return GetTopChangesAsJSON( + test_case_filter, regressions, last, timescope, premature, + limit) + + # --- 'namemappings' --------------------------------- + elif cmd == "namemappings": + return GetNameMappingsAsJSON() + + # --- 'testcaseswithchanges' --------------------------------- + elif cmd == "testcaseswithchanges": + return GetTestCasesWithChangesAsJSON() + + # No match: printUsageError() sys.exit(1) diff --git a/scripts/gettestcaseswithchanges.py b/scripts/gettestcaseswithchanges.py new file mode 100644 index 0000000..c1d44c2 --- /dev/null +++ b/scripts/gettestcaseswithchanges.py @@ -0,0 +1,30 @@ +import sys, json +from dbaccess import execQuery +from misc import printJSONHeader + + +class GetTestCasesWithChanges: + + def __init__(self): + pass + + def execute(self): + self.test_cases = execQuery( + "SELECT value FROM (SELECT DISTINCT testCaseId FROM change)" + " AS foo, testCase WHERE testCase.id = testCaseId" + " ORDER BY value", ()) + + # Flatten one level: + self.test_cases = ( + [item for sublist in self.test_cases for item in sublist]) + + self.writeOutput() + + def writeOutputAsJSON(self): + printJSONHeader() + json.dump({ 'testCases': self.test_cases }, sys.stdout) + + +class GetTestCasesWithChangesAsJSON(GetTestCasesWithChanges): + def writeOutput(self): + self.writeOutputAsJSON() diff --git a/scripts/gettopchanges.py b/scripts/gettopchanges.py new file mode 100644 index 0000000..1b7282a --- /dev/null +++ b/scripts/gettopchanges.py @@ -0,0 +1,181 @@ +import sys, json, calendar, time +from dbaccess import execQuery, database +from misc import ( + textToId, idToText, getContext, getTimestampFromContext, getSnapshots, + benchmarkToComponents, printJSONHeader) + + + +# Gets the top changes for a specific context by considering only the +# last change in each time series. +# +# An additional filter is applied by only considering results from +# test cases matching test_case_ids. +# +# NOTE: The result may contain at most one change from a given time series. +def getTopChangesForContext_last( + host_id, platform_id, branch_id, regressions, premature, limit, + test_case_ids): + + query = ( + "SELECT change.benchmarkId, change.metricId, sha1Id, last_timestamp, ") + query += ("greatest(score, premature_score)" if premature else "score") + query += ( + " AS final_score" + " FROM" + " (SELECT benchmarkId, metricId, max(timestamp) AS last_timestamp" + " FROM change" + " WHERE hostId = %s" + " AND platformId = %s" + " AND branchId = %s" + " AND regression = %s") + args = [host_id, platform_id, branch_id, regressions] + + if len(test_case_ids) > 0: + query += ( + " AND testCaseId IN (%s" + ", %s"*(len(test_case_ids) - 1) + ")" + ) + args += test_case_ids + + query += ( + " GROUP BY benchmarkId, metricId) AS last_change" + " , change" + " WHERE last_change.benchmarkId = change.benchmarkId" + " AND last_change.metricId = change.metricId" + " AND change.timestamp = last_timestamp" + " ORDER BY final_score DESC, last_timestamp DESC" + " LIMIT %s" + ) + args.append(limit) + + return execQuery(query, args) + + +# Gets the top changes for a specific context by considering all +# changes in the given time scope. +# +# An additional filter is applied by only considering results from +# test cases matching test_case_ids. +# +# NOTE: The result may contain any number of changes from a given time +# series. +def getTopChangesForContext_timeScope( + host_id, platform_id, branch_id, regressions, premature, limit, + test_case_ids, lo_timestamp): + + query = "SELECT benchmarkId, metricId, sha1Id, timestamp, " + query += ("greatest(score, premature_score)" if premature else "score") + query += ( + " AS final_score" + " FROM change" + " WHERE hostId = %s" + " AND platformId = %s" + " AND branchId = %s" + " AND regression = %s" + " AND timestamp >= %s") + args = [host_id, platform_id, branch_id, regressions, lo_timestamp] + + if len(test_case_ids) > 0: + query += ( + " AND testCaseId IN (%s" + ", %s"*(len(test_case_ids) - 1) + ")" + ) + args += test_case_ids + + query += ( + " ORDER BY final_score DESC, timestamp DESC" + " LIMIT %s" + ) + args.append(limit) + + return execQuery(query, args) + + +# Returns test case IDs corresponding to the names in test_case_filter. +def getTestCaseIdsFromFilter(test_case_filter): + return (execQuery( + "SELECT id FROM testCase" + " WHERE value IN (%s" + ", %s"*(len(test_case_filter) - 1) + ")", + tuple(test_case_filter)) + if ((test_case_filter != None) and (len(test_case_filter) > 0)) + else ()) + + +class GetTopChanges: + + def __init__( + self, test_case_filter, regressions, last, timescope, premature, limit): + self.test_case_filter = test_case_filter + self.regressions = regressions + self.last = last + self.timescope = timescope + self.premature = premature + self.limit = limit + + + # Gets the top changes for a specific context. + def getTopChangesForContext( + self, host_id, platform_id, branch_id, lo_timestamp): + + test_case_ids = getTestCaseIdsFromFilter(self.test_case_filter) + + return ( + getTopChangesForContext_last( + host_id, platform_id, branch_id, self.regressions, + self.premature, self.limit, test_case_ids) + if self.last else + getTopChangesForContext_timeScope( + host_id, platform_id, branch_id, self.regressions, + self.premature, self.limit, test_case_ids, lo_timestamp) + ) + + + # Gets the top changes for all contexts. + def getTopChangesForAllContexts(self): + + # Compute lowest timestamp (secs since 1970) in time scope (days ago): + curr_timestamp = calendar.timegm(time.gmtime()) + secs_in_day = 24 * 60 * 60 + lo_timestamp = ( + -1 if (self.timescope < 0) else + curr_timestamp - self.timescope * secs_in_day) + + context_ids = execQuery( + "SELECT DISTINCT hostId, platformId, branchId FROM context " + "ORDER BY hostId, platformId, branchId", ()) + + contexts = [] + + for host_id, platform_id, branch_id in context_ids: + top_changes = self.getTopChangesForContext( + host_id, platform_id, branch_id, lo_timestamp) + contexts.append({ + "hostId": host_id, + "platformId": platform_id, + "branchId": branch_id, + "topchanges": top_changes + }) + + return contexts + + + def execute(self): + self.contexts = self.getTopChangesForAllContexts() + self.writeOutput() + + + def writeOutputAsJSON(self): + printJSONHeader() + json.dump({ + 'database': database(), + 'regressions': self.regressions, + 'last': self.last, + 'timescope': self.timescope, + 'premature': self.premature, + 'limit': self.limit, + 'contexts': self.contexts + }, sys.stdout) + + +class GetTopChangesAsJSON(GetTopChanges): + def writeOutput(self): + self.writeOutputAsJSON() diff --git a/scripts/listcontexts.py b/scripts/listcontexts.py index 4aae8bc..a78daa5 100644 --- a/scripts/listcontexts.py +++ b/scripts/listcontexts.py @@ -6,15 +6,8 @@ from misc import idToText, printJSONHeader class ListContexts: - def __init__(self, ranked_only): - self.ranked_only = ranked_only - - - def rankingsExist(self, contextId): - rankings = execQuery( - "SELECT id FROM ranking WHERE context2Id = %s LIMIT 1", - (contextId,)) - return len(rankings) > 0 + def __init__(self): + pass def execute(self): @@ -36,44 +29,29 @@ class ListContexts: id, host, platform, branch, sha1, timestamp = contexts[0] curHost, curPlatform, curBranch = host, platform, branch - - rankings_exist = self.rankingsExist(id) - rankings_exist_count = 1 if rankings_exist else 0 - - curSnapshots = [] - if (not self.ranked_only) or rankings_exist: - curSnapshots = [ - (idToText('sha1', sha1), timestamp, 1 if rankings_exist else 0)] + curSnapshots = [(idToText('sha1', sha1), timestamp)] for (id, host, platform, branch, sha1, timestamp) in contexts[1:]: if (host, platform, branch) != (curHost, curPlatform, curBranch): - if (not self.ranked_only) or (rankings_exist_count > 0): - assert len(curSnapshots) > 0 - self.contexts.append({ - 'host' : idToText('host', curHost), - 'platform' : idToText('platform', curPlatform), - 'branch' : idToText('branch', curBranch), - 'snapshots' : curSnapshots - }) - - rankings_exist_count = 0 + self.contexts.append({ + 'host' : idToText('host', curHost), + 'platform' : idToText('platform', curPlatform), + 'branch' : idToText('branch', curBranch), + 'snapshots' : curSnapshots + }) curHost, curPlatform, curBranch = host, platform, branch curSnapshots = [] - rankings_exist = self.rankingsExist(id) - if rankings_exist: - rankings_exist_count = rankings_exist_count + 1 - if (not self.ranked_only) or rankings_exist: - curSnapshots.append( - (idToText('sha1', sha1), timestamp, - 1 if rankings_exist else 0)) + curSnapshots.append((idToText('sha1', sha1), timestamp)) self.writeOutput() + def writeOutputAsJSON(self): printJSONHeader() json.dump({ 'contexts' : self.contexts }, sys.stdout) + class ListContextsAsJSON(ListContexts): def writeOutput(self): self.writeOutputAsJSON() diff --git a/scripts/misc.py b/scripts/misc.py index b3a2274..8bd8516 100644 --- a/scripts/misc.py +++ b/scripts/misc.py @@ -45,6 +45,33 @@ def textToId(table, text): # ### 2 B DOCUMENTED! +def findOrInsertId(table, value, *args): + + query_result = execQuery( + "SELECT id FROM " + table + " WHERE value = %s", (value,)) + if len(query_result) == 1: + # Found, so return ID: + return query_result[0][0] + + # Not found, so insert: + query = "INSERT INTO " + table + " (value" + for i in range(0, len(args), 2): + query += ", " + args[i] + query += ") VALUES (%s" + values = [value] + for i in range(0, len(args), 2): + query += ", %s" + values.append(args[i + 1]) + + # ... and retrieve ID: + query += ") RETURNING id" + query_result = execQuery(query, values) + + assert len(query_result) == 1 + return query_result[0][0] + + +# ### 2 B DOCUMENTED! # Maybe also rename to lowerIsBetter() ? (but note that a global function with # that name already exists in uploadresults.py) def metricIdToLowerIsBetter(metric_id): @@ -64,6 +91,13 @@ def metricIdToLowerIsBetter(metric_id): # Returns the non-negative ID of the given context, or -1 if not found. def getContext(host_id, platform_id, branch_id, sha1_id): + global contextIdCache + if not 'contextIdCache' in globals(): + contextIdCache = {} + + if (host_id, platform_id, branch_id, sha1_id) in contextIdCache: + return contextIdCache[host_id, platform_id, branch_id, sha1_id] + result = execQuery( "SELECT id FROM context" " WHERE hostId = %s" @@ -72,9 +106,10 @@ def getContext(host_id, platform_id, branch_id, sha1_id): " AND sha1Id = %s" "LIMIT 1", (host_id, platform_id, branch_id, sha1_id)) - if len(result): - return result[0][0] - return -1 + result = result[0][0] if len(result) else -1 + contextIdCache[host_id, platform_id, branch_id, sha1_id] = result + + return result # Returns the test case, test function, and data tag components of @@ -100,32 +135,46 @@ def getTimestampFromContext(context_id): # Finds snapshots that match a host/platform/branch combination and that # lie within the range -# [sha11, sha12] if sha12_id >= 0, or -# [sha11, +inf) if sha12_ is < 0. +# [sha11, sha12] if both sha11_id and sha12_id are >= 0, or +# ( -inf, sha12] if only sha11_id is < 0, or +# [sha11, +inf) if only sha12_id is < 0, or +# ( -inf, +inf) if both sha11_id and sha2_id are < 0 +# # Returns a chronologically order n-tuple of 2-tuples: # (sha1, first upload timestamp). def getSnapshots(host_id, platform_id, branch_id, sha11_id, sha12_id): - timestamp1 = execQuery( + + timestamp1 = (execQuery( "SELECT EXTRACT(EPOCH FROM timestamp)::INT FROM context" " WHERE hostId = %s" " AND platformId = %s" " AND branchId = %s" " AND sha1Id = %s", (host_id, platform_id, branch_id, sha11_id))[0][0] - if sha12_id >= 0: - timestamp2 = execQuery( - "SELECT EXTRACT(EPOCH FROM timestamp)::INT FROM context" - " WHERE hostId = %s" - " AND platformId = %s" - " AND branchId = %s" - " AND sha1Id = %s", - (host_id, platform_id, branch_id, sha12_id))[0][0] + if (sha11_id >= 0) else -1) + + timestamp2 = (execQuery( + "SELECT EXTRACT(EPOCH FROM timestamp)::INT FROM context" + " WHERE hostId = %s" + " AND platformId = %s" + " AND branchId = %s" + " AND sha1Id = %s", + (host_id, platform_id, branch_id, sha12_id))[0][0] + if (sha12_id >= 0) else -1) + + prefix = "AND EXTRACT(EPOCH FROM timestamp)::INT" + + if (timestamp1 == -1) and (timestamp2 == -1): + range_expr = "" + elif (timestamp1 >= 0) and (timestamp2 < 0): + range_expr = ("%s >= %d" % (prefix, timestamp1)) + elif (timestamp1 < 0) and (timestamp2 >= 0): + range_expr = ("%s <= %d" % (prefix, timestamp2)) + else: # Ensure chronological order: if timestamp1 > timestamp2: timestamp1, timestamp2 = timestamp2, timestamp1 - range_expr = "BETWEEN %d AND %d" % (timestamp1, timestamp2) - else: - range_expr = ">= %d" % timestamp1 + range_expr = ("%s BETWEEN %d AND %d" % (prefix, timestamp1, timestamp2)) # Each distinct SHA-1 that occurs for this host/platform/branch # combination may occur multiple times with different upload times. @@ -138,8 +187,7 @@ def getSnapshots(host_id, platform_id, branch_id, sha11_id, sha12_id): " FROM context" " WHERE hostId = %s" " AND platformId = %s" - " AND branchId = %s" - " AND EXTRACT(EPOCH FROM timestamp)::INT " + range_expr + + " AND branchId = %s " + range_expr + " ORDER BY timestamp ASC", (host_id, platform_id, branch_id)) @@ -169,48 +217,8 @@ def getAllSnapshots(host_id, platform_id, branch_id, reverse = False): return tuple(snapshots) -# Returns the (SHA-1 ID, timestamp) pair associated with the most recent -# rankings computed for the given host/platform/branch combination, or -# (-1, -1) if no match is found. -def getLastRankingSnapshot(host_id, platform_id, branch_id): - result = execQuery( - "SELECT matchingcontext.sha1id, EXTRACT(EPOCH FROM timestamp)::INT" - " FROM ranking," - " (SELECT id, sha1Id, timestamp" - " FROM context" - " WHERE hostId = %s" - " AND platformId = %s" - " AND branchId = %s) AS matchingContext" - " WHERE context2Id = matchingContext.id" - " ORDER BY timestamp DESC LIMIT 1", - (host_id, platform_id, branch_id)) - if len(result): - return result[0] - return -1, -1 - - -# For the given host/platform/branch combination, this function returns -# all contexts for which rankings exist. The return value is a list of -# (context ID, timestamp) pairs sorted in descending order on timestamp -# (latest timestamp first). -def getRankingContexts(host_id, platform_id, branch_id): - result = execQuery( - "SELECT DISTINCT matchingcontext.id," - " EXTRACT(EPOCH FROM timestamp)::INT AS etimestamp" - " FROM ranking," - " (SELECT id, sha1Id, timestamp" - " FROM context" - " WHERE hostId = %s" - " AND platformId = %s" - " AND branchId = %s) AS matchingContext" - " WHERE context2Id = matchingContext.id" - " ORDER BY etimestamp DESC", - (host_id, platform_id, branch_id)) - return result - - -# Retrieves the time series of valid median results for the given -# benchmark/metric combination. Only the part of the time series that +# Retrieves the time series + additional stats of valid median results for +# the given benchmark/metric combination. Only the part of the time series that # is within the selected snapshot interval is considered. # # Returns a 7-tuple: @@ -355,6 +363,61 @@ def getTimeSeries( ms, lsd) +# Retrieves the time series of valid median results for the given +# benchmark/metric combination within the given contexts. +# Only the part of the time series that is within the selected snapshot +# interval is considered. +# +# Returns an n-tuple of 2-tuples: +# +# ( +# <corresponding index in the 'contexts' list>, +# <median of valid observations or -1 if all obs. are invalid> +# ) +# +def getBasicTimeSeries(contexts, benchmark_id, metric_id): + + # Fetch raw values: + assert len(contexts) > 0 + raw_values = (execQuery( + "SELECT value, valid, contextId FROM result" + " WHERE contextId IN (%s" + ", %s"*(len(contexts) - 1) + ")" + + " AND benchmarkId = %s" + " AND metricId = %s" + " ORDER BY contextId", + tuple(contexts) + (benchmark_id, metric_id)) + + [(-1, -1, -1)]) # Note sentinel item + + # Compute per-sample stats: + curr_context_id = -1 + valid_and_positive_sample = [] + median_obs_map = {} + # Loop over all observations (which are grouped on sample; + # note the 1-1 correspondence between samples and contexts): + for obs, valid, context_id in raw_values: + if context_id != curr_context_id: + # A new sample has been collected, so register it and + # prepare for the next one: + median_obs = stats.medianscore(valid_and_positive_sample) if ( + len(valid_and_positive_sample) > 0) else -1 + median_obs_map[curr_context_id] = median_obs + valid_and_positive_sample = [] + curr_context_id = context_id + # Append a valid and positive observation to the current sample: + if valid and (obs > 0): + valid_and_positive_sample.append(obs) + + # Order chronologically: + ts = [] + index = 0 + for context in contexts: + if context in median_obs_map: + ts.append((index, median_obs_map[context])) + index = index + 1 + + return ts + + # Returns the factor by which val improves over base_val by taking the # lower_is_better property into consideration. # Example: base_val = 10 and val = 20 results in 0.5 if lower_is_better is true, @@ -369,7 +432,7 @@ def metricAdjustedRatio(base_val, val, lower_is_better): # Whether a change is significant depends on the difftol argument. # Only positive values are considered. # -# The output is an n-tuple of 7-tuples, one per change: +# The output is a list of 7-tuples, one per change: # # 1: Base index, i.e. the index in the time series that contains the base # value used to compute the change. @@ -545,6 +608,84 @@ def getChanges(time_series, lower_is_better, difftol, durtolmin, durtolmax): return tuple(changes) +# Extracts the (significant) changes of all time series in a given +# host/platform/branch combination. +# +# The output is a list of 3-tuples, one per time series having at least one +# change: +# +# 1: Benchmark ID. +# 2: Metric ID. +# 3: The changes as a list of 9-tuples. The first 7 elements correspond to +# the output from getChanges() (documented elsewhere). Element 8 +# and 9 are the SHA1 ID and timestamp corresponding to the change. +# +def getAllChanges( + host_id, platform_id, branch_id, difftol, durtolmin, durtolmax, + progress_func = None, progress_arg = None): + + if progress_func != None: + progress_func(0.0, progress_arg) + + # Get all snapshots matching the host/platform/branch combination: + snapshots = getAllSnapshots(host_id, platform_id, branch_id) + + # Get the list of contexts that corresponds to these snapshots in this + # host/platform/branch combination: + contexts = [] + for sha1_id, timestamp in snapshots: + contexts.append(getContext(host_id, platform_id, branch_id, sha1_id)) + + if len(contexts) == 0: + return [] + + # Get all distinct benchmark/metric combinations that match the + # host/platform/branch context and are within the selected snapshot + # interval. Each such combination corresponds to a time series. + tseries_list = execQuery( + "SELECT DISTINCT benchmarkId, metricId FROM result" + " WHERE contextId IN (%s" + ", %s"*(len(contexts) - 1) + ")", +# " WHERE contextId IN (%s" + ", %s"*(len(contexts) - 1) + ") LIMIT 10", + contexts) + + changes = [] + + # Loop over time series: + if progress_func != None: + i = 0 + for benchmark_id, metric_id in tseries_list: + + # Get the time series (without extra stats): + time_series = getBasicTimeSeries(contexts, benchmark_id, metric_id) + + # Extract the significant changes: + basic_tschanges = getChanges( + time_series, metricIdToLowerIsBetter(metric_id), difftol, + durtolmin, durtolmax) + # ... add the SHA-1 and timestamp to each item: + tschanges = [] + for change in basic_tschanges: + index = time_series[change[1]][0] + sha1_id = snapshots[index][0] + timestamp = snapshots[index][1] + tschanges.append(change + (sha1_id, timestamp)) + + if len(tschanges) > 0: + changes.append((benchmark_id, metric_id, tschanges)) + + if progress_func != None: + i = i + 1 + divisor = len(tseries_list) // 100 # report at most 100 times + if (divisor > 0) and ((i % divisor) == 0): + perc_done = (i / float(len(tseries_list))) * 100.0 + progress_func(perc_done, progress_arg) + + if progress_func != None: + progress_func(100.0, progress_arg) + + return changes + + # ### 2 B DOCUMENTED! def getTimeSeriesMiscStats(time_series, changes, snapshots, stats): if len(changes) > 0: @@ -654,6 +795,170 @@ def getBMTimeSeriesStatsList( return tuple(bmstats_list) +# Returns the score for the most recent change in 'changes'. +# If 'regression' is true, only regressions are considered, and otherwise +# only improvements. +# If 'premature' is true, the premature score (not considering post-change +# durability) is returned instead of the regular score. +# +# Returns -1 if no score is found. +# + + +# OBSOLETE ??? + + +def getLastChangeScore(changes, regression, premature): + pass + + +# Returns the highest score for any change in a certain time interval. +# +# The time interval is defined like this: <2 B DONE!> +# +# If 'regression' is true, only regressions are considered, and otherwise +# only improvements. +# If 'premature' is true, the premature score (not considering post-change +# durability) is returned instead of the regular score. +# +# Returns -1 if no score is found. +# + + +# +# OBSOLETE ??? + + +def getHighestChangeScore(changes, regression, premature, days): + pass + + +# Computes change scores for each time series (benchmark/metric combination) +# of the given host/platform/branch combination. +# +# Change types: +# - most recent change (i.e. significant regression or improvement) +# - strongest change last n days, for n in {7, 30, 180, and -1 (infinite)} +# ... and all of these with and without inclusion of premature changes. +# +# ADD MORE DOCS HERE ... 2 B DONE! +# + + +# OBSOLETE ??? + + +# def getChangeScores( +# host_id, platform_id, branch_id, snapshots, test_case_filter, +# difftol, durtolmin, durtolmax, progress_func = None, progress_arg = None): + +# if progress_func != None: +# progress_func(0.0, progress_arg) + +# contexts = [] +# for sha1_id, timestamp in snapshots: +# contexts.append(getContext(host_id, platform_id, branch_id, sha1_id)) + +# # Get all distinct benchmark/metric combinations that match the +# # host/platform/branch context and are within the selected snapshot +# # interval. Each such combination corresponds to a time series. +# assert len(contexts) > 0 +# tseries = execQuery( +# "SELECT DISTINCT benchmarkId, metricId FROM result" +# " WHERE contextId IN (%s" + ", %s"*(len(contexts) - 1) + ")", +# contexts) + +# scores = [] + +# # Loop over time series: +# if progress_func != None: +# i = 0 +# #for benchmark_id, metric_id in tseries[800:810]: +# for benchmark_id, metric_id in tseries: + +# benchmark = idToText("benchmark", benchmark_id) +# #if benchmark != "tst_qmetaobject:indexOfMethod(_q_columnsAboutToBeRemoved(QModelIndex,int,int))": +# # continue + +# test_case, test_function, data_tag = ( +# benchmarkToComponents(benchmark)) + +# # Skip this time series if it doesn't match the test case filter: +# if ((test_case_filter != None) +# and (not test_case in test_case_filter)): +# continue + +# # Get the time series (without extra stats): +# time_series = getBasicTimeSeries( +# host_id, platform_id, branch_id, snapshots, benchmark_id, metric_id) + +# # Extract the significant changes: +# changes = getChanges( +# time_series, metricIdToLowerIsBetter(metric_id), difftol, +# durtolmin, durtolmax) + +# tsscores = {} + +# tsscores["benchmark_id"] = benchmark_id +# tsscores["metric_id"] = metric_id + +# tsscores["regr_last"] = getLastChangeScore(changes, True, False) +# tsscores["regr_last_pmt"] = getLastChangeScore(changes, True, True) +# tsscores["impr_last"] = getLastChangeScore(changes, False, False) +# tsscores["impr_last_pmt"] = getLastChangeScore(changes, False, True) + +# tsscores["regr_7"] = getHighestChangeScore( +# changes, True, False, 7) +# tsscores["regr_7_pmt"] = getHighestChangeScore( +# changes, True, True, 7) +# tsscores["impr_7"] = getHighestChangeScore( +# changes, False, False, 7) +# tsscores["impr_7_pmt"] = getHighestChangeScore( +# changes, False, True, 7) + +# tsscores["regr_30"] = getHighestChangeScore( +# changes, True, False, 30) +# tsscores["regr_30_pmt"] = getHighestChangeScore( +# changes, True, True, 30) +# tsscores["impr_30"] = getHighestChangeScore( +# changes, False, False, 30) +# tsscores["impr_30_pmt"] = getHighestChangeScore( +# changes, False, True, 30) + +# tsscores["regr_180"] = getHighestChangeScore( +# changes, True, False, 180) +# tsscores["regr_180_pmt"] = getHighestChangeScore( +# changes, True, True, 180) +# tsscores["impr_180"] = getHighestChangeScore( +# changes, False, False, 180) +# tsscores["impr_180_pmt"] = getHighestChangeScore( +# changes, False, True, 180) + +# tsscores["regr_all"] = getHighestChangeScore( +# changes, True, False, -1) +# tsscores["regr_all_pmt"] = getHighestChangeScore( +# changes, True, True, -1) +# tsscores["impr_all"] = getHighestChangeScore( +# changes, False, False, -1) +# tsscores["impr_all_pmt"] = getHighestChangeScore( +# changes, False, True, -1) + +# scores.append(tsscores) + +# if progress_func != None: +# i = i + 1 +# divisor = len(tseries) // 100 # report at most 100 times +# if (divisor > 0) and ((i % divisor) == 0): +# perc_done = (i / float(len(tseries))) * 100.0 +# progress_func(perc_done, progress_arg) + +# if progress_func != None: +# progress_func(100.0, progress_arg) + +# return scores + + + # Returns True iff s is a valid SHA-1 string. def isValidSHA1(s): def containsOnlyHexDigits(s): diff --git a/scripts/updateallchanges.py b/scripts/updateallchanges.py new file mode 100755 index 0000000..e5d81c8 --- /dev/null +++ b/scripts/updateallchanges.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python + +""" +This script invokes updatechanges.py for all host/platform/branch combinations. +""" + +import sys +from subprocess import Popen, PIPE +from dbaccess import setDatabase, execQuery, commit +from misc import getOptions, idToText + + +# --- BEGIN Global functions ---------------------------------------------- + +def printUsage(): + sys.stderr.write( + "usage: " + sys.argv[0] + " --help | [--dbhost H --dbport P] --db D\n") + + +def printVerboseUsage(): + printUsage() + sys.stderr.write("\noptions:\n") + sys.stderr.write( + " --help: This help.\n") + sys.stderr.write( + " --dbhost: The database server host (overriding the default).\n") + sys.stderr.write( + " --dbport: The database server port (overriding the default).\n") + sys.stderr.write( + " --db: The database. One of 'bm' or 'bm-dev' (the latter " + "intended for experimentation).\n") + + +# Executes the external updatechanges.py script with appropriate arguments. +def execUpdateChanges(host, platform, branch, options): + + cmd = [ + "updatechanges.py", "--db", options["db"], "--host", host, + "--platform", platform, "--branch", branch, "--noprogress", "true"] + if "dbhost" in options: + cmd += ["--dbhost", options["dbhost"]] + if "dbport" in options: + cmd += ["--dbport", options["dbport"]] + + sys.stdout.write( + "\nupdating changes for " + host + " / " + platform + " / " + branch + + " ...\n") + sys.stdout.flush() + + p = Popen(cmd, stdout = PIPE, stderr = PIPE) + stdout, stderr = p.communicate() + if (p.returncode != 0): + sys.stdout.write("failed to execute command '" + str(cmd) + "':\n") + sys.stdout.write(" return code: " + str(p.returncode) + "\n") + sys.stdout.write(" stdout: >" + stdout.strip() + "<\n") + sys.stdout.write(" stderr: >" + stderr.strip() + "<\n") + else: + sys.stdout.write("updatechanges.py executed successfully:\n") + sys.stdout.write(" return code: " + str(p.returncode) + "\n") + sys.stdout.write(" stdout: >" + stdout.strip() + "<\n") + sys.stdout.write(" stderr: >" + stderr.strip() + "<\n") + +# --- END Global functions ---------------------------------------------- + + +# --- BEGIN Main program ---------------------------------------------- + +options, http_get = getOptions() + +if "help" in options: + printVerboseUsage() + sys.exit(1) + +if not ("db" in options): + printUsage() + sys.exit(1) + +setDatabase( + options["dbhost"] if "dbhost" in options else None, + options["dbport"] if "dbport" in options else None, + options["db"]) + +hpb_ids = execQuery( + "SELECT DISTINCT hostId, platformId, branchId FROM context " + "ORDER BY hostId, platformId, branchId", ()) + +for host_id, platform_id, branch_id in hpb_ids: + execUpdateChanges( + idToText("host", host_id), + idToText("platform", platform_id), + idToText("branch", branch_id), + options) + +sys.exit(0) + +# --- END Main program ---------------------------------------------- diff --git a/scripts/updatechanges.py b/scripts/updatechanges.py new file mode 100755 index 0000000..3e1c8dd --- /dev/null +++ b/scripts/updatechanges.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python + +""" +This script is intended to be executed whenever a new snapshot is complete +for a host/platform/branch combination. + +The script registers - in the 'change' table - all current changes of all +time series of the host/platform/branch combination in question (wiping all +existing changes first). + +The 'Change Summary' web page can then populate its internal tables +directly from the 'change' table. +""" + + +import sys +from dbaccess import setDatabase, execQuery, commit +from misc import getOptions, textToId, getAllChanges + + +# --- BEGIN Global functions ---------------------------------------------- + +def printUsage(): + sys.stderr.write( + "usage: " + sys.argv[0] + + " --help | [--dbhost H] [--dbport P] --db D --host H --platform P " + "--branch B [--noprogress NP]\n") + +def printVerboseUsage(): + printUsage() + sys.stderr.write("\noptions:\n") + sys.stderr.write( + " --help: This help.\n") + sys.stderr.write( + " --dbhost: The database server host (overriding the default).\n") + sys.stderr.write( + " --dbport: The database server port (overriding the default).\n") + sys.stderr.write( + " --db: The database. One of 'bm' or 'bm-dev' (the latter " + "intended for experimentation).\n") + sys.stderr.write( + " --host: The physical machine on which the results were " + "produced (e.g. barbarella or 172.24.90.79).\n") + sys.stderr.write( + "--platform: The OS/compiler/architecture combination " + "(e.g. linux-g++-32).\n") + sys.stderr.write( + " --branch: The product branch (e.g. 'qt 4.6', 'qt 4.7', or " + "'qt master').\n") + sys.stderr.write( + " --noprogress: Specify \'true\' to disable progress indicator.\n") + + +# ### 2 B DOCUMENTED! +def printProgress(p, lead): + sys.stdout.write(lead + " ... (" + "{0:.2f}".format(p) + " %)\r") + sys.stdout.flush() + + +# ### 2 B DOCUMENTED! +# NOTE: This function is currently duplicated elsewhere in JavaScipt! +def changeMagnitudeScore(change): + max_change = 2.0 + abs_change = (1.0 / change) if change < 1 else change + return (min(abs_change, max_change) - 1.0) / (max_change - 1.0) + + +# Updates the 'changes' table in the current database with all changes in +# all time series (i.e. benchmark/metric combinations) of the given +# host/platform/branch combination. Progress will be written standard output +# iff no_progress is False. +# +# The algorithm is effectively this: +# - Compute the current changes. +# - Delete all rows matching the given host/platform/branch combination. +# - Add a row for each individual change. +# +def updateChanges(host_id, platform_id, branch_id, no_progress): + + # Hardcode tolerances here for now: + difftol = 1.1 + durtolmin = 3 + durtolmax = 10 + + # Get all changes for this host/platform/branch combination: + if no_progress: + sys.stdout.write( + "getting all changes for this host/platform/branch " + "combination ... ") + changes = getAllChanges( + host_id, platform_id, branch_id, difftol, durtolmin, durtolmax, + None if no_progress else printProgress, + "getting all changes for this host/platform/branch combination") + + if no_progress: + sys.stdout.write("done") + sys.stdout.write( + "\n" + str(len(changes)) + " time series with changes found\n") + + if len(changes) == 0: + sys.stderr.write("error: no time series with changes found\n") + sys.exit(1) + + + # Store the changes in the 'change' table: + sys.stdout.write("storing to 'change' table ...") + sys.stdout.flush() + + # ... delete all rows matching this host/platform/branch combination: + execQuery( + "DELETE FROM change" + " WHERE hostId = %s" + " AND platformId = %s" + " AND branchId = %s", (host_id, platform_id, branch_id), False) + + # ... insert rows for the new changes: + query = ( + "INSERT INTO change" + " (benchmarkId, testCaseId, metricId, hostId, platformId, branchId," + " sha1Id, timestamp, regression, score, premature_score) VALUES ") + args = [] + + for benchmark_id, metric_id, tschanges in changes: + + test_case_id = execQuery( + "SELECT testCaseId FROM benchmark WHERE id = %s", + (benchmark_id,))[0][0]; + + for change in tschanges: + + ratio = change[2] + regression = ratio < 1.0 + magn_score = changeMagnitudeScore(ratio) + gsep_score = change[3] + lsep_score = change[4] + dur_score1 = change[5] + dur_score2 = change[6] + premature_score = magn_score * gsep_score * lsep_score * dur_score1 + score = premature_score * dur_score2 + sha1_id = change[7] + timestamp = change[8] + + if len(args) > 0: + query += ", " + query += ("(%s" + ", %s"*10 + ") ") + args += ( + benchmark_id, test_case_id, metric_id, host_id, platform_id, + branch_id, sha1_id, timestamp, regression, score, + premature_score) + + execQuery(query, args, False) + + sys.stdout.write("done\n") + sys.stdout.flush() + + + +# --- END Global functions ---------------------------------------------- + + +# --- BEGIN Main program ---------------------------------------------- + +options, http_get = getOptions() + +if "help" in options: + printVerboseUsage() + sys.exit(1) + +if (not ("db" in options and "host" in options and "platform" in options and + "branch" in options)): + printUsage() + sys.exit(1) + +setDatabase( + options["dbhost"] if "dbhost" in options else None, + options["dbport"] if "dbport" in options else None, + options["db"]) + +host_id = textToId("host", options["host"]) +if host_id == -1: + sys.stderr.write("error: no such host: " + options["host"] + "\n") + sys.exit(1) +platform_id = textToId("platform", options["platform"]) +if platform_id == -1: + sys.stderr.write("error: no such platform: " + options["platform"] + "\n") + sys.exit(1) +branch_id = textToId("branch", options["branch"]) +if branch_id == -1: + sys.stderr.write("error: no such branch:" + options["branch"] + "\n") + sys.exit(1) + +updateChanges( + host_id, platform_id, branch_id, + ("noprogress" in options) and ( + (options["noprogress"] == "1") + or (options["noprogress"].lower() == "true"))) + +# Write to database: +commit() + +# --- END Main program ---------------------------------------------- diff --git a/scripts/uploadresults.py b/scripts/uploadresults.py index a50285c..89f97ab 100755 --- a/scripts/uploadresults.py +++ b/scripts/uploadresults.py @@ -5,7 +5,8 @@ from subprocess import Popen, PIPE from xml.dom.minidom import parse, getDOMImplementation from dbaccess import setDatabase, execQuery, commit from misc import ( - getOptions, textToId, idToText, isValidSHA1, getContext, getAllSnapshots) + getOptions, textToId, idToText, findOrInsertId, isValidSHA1, getContext, + getAllSnapshots) # --- BEGIN Global functions ---------------------------------------------- @@ -182,33 +183,6 @@ def extractResults(file): return results -# ### 2 B DOCUMENTED! -def findOrInsertId(table, value, *args): - - query_result = execQuery( - "SELECT id FROM " + table + " WHERE value = %s", (value,)) - if len(query_result) == 1: - # Found, so return ID: - return query_result[0][0] - - # Not found, so insert: - query = "INSERT INTO " + table + " (value" - for i in range(0, len(args), 2): - query += ", " + args[i] - query += ") VALUES (%s" - values = [value] - for i in range(0, len(args), 2): - query += ", %s" - values.append(args[i + 1]) - - # ... and retrieve ID: - query += ") RETURNING id" - query_result = execQuery(query, values) - - assert len(query_result) == 1 - return query_result[0][0] - - # Uploads a set of results to the database. def uploadToDatabase(host, platform, branch, sha1, results): @@ -237,7 +211,9 @@ def uploadToDatabase(host, platform, branch, sha1, results): benchmark = ( result['testCase'] + ":" + result['testFunction'] + "(" + str(result['dataTag']) + ")") - benchmarkId = findOrInsertId("benchmark", benchmark) + testCaseId = findOrInsertId("testCase", result['testCase']) + benchmarkId = findOrInsertId( + "benchmark", benchmark, "testCaseId", testCaseId) metricId = findOrInsertId( "metric", result['metric'], "lowerIsBetter", @@ -254,20 +230,6 @@ def uploadToDatabase(host, platform, branch, sha1, results): commit() -# Returns True iff rankings exist for the given context. -def rankingsExist(options): - context_id = getContext( - textToId('host', options["host"]), - textToId('platform', options["platform"]), - textToId('branch', options["branch"]), - textToId('sha1', options["sha1"])) - - matches = execQuery( - "SELECT id FROM ranking WHERE context2Id = %s LIMIT 1", (context_id,)) - - return len(matches) > 0 - - # Returns the context ID if found, otherwise -1: def getContextIdFromNames(options): host_id = textToId("host", options["host"]) @@ -297,35 +259,12 @@ def contextComplete(options): return sample_size >= max_sample_size -# Executes the external computerankings.py script with appropriate arguments. -# If new_context is True, this snapshot is assumed to be the first one in -# this time series (host/platform/branch combination), and no results have been -# uploaded for this context yet. In that case, rankings will instead be -# computed for the latest existing snapshot in this time series. -def execComputeRankings(options, new_context): - - if new_context: - # Attempt to use the latest available snapshot for this - # host/platform/branch combination as the actual snapshot: - host_id = textToId("host", options["host"]) - platform_id = textToId("platform", options["platform"]) - branch_id = textToId("branch", options["branch"]) - snapshots = getAllSnapshots(host_id, platform_id, branch_id) - if len(snapshots) > 0: - actual_snapshot = idToText("sha1", snapshots[-1][0]) - if actual_snapshot == options["sha1"]: - sys.stderr.write( - "error: context unexpectedly exists in database\n") - sys.exit(1) - else: - return # special case when no snapshots exist yet - else: - actual_snapshot = options["sha1"] - +# Executes the external updatechanges.py script with appropriate arguments. +def execUpdateChanges(options): cmd = [ - "computerankings.py", "--db", options["db"], "--host", options["host"], + "updatechanges.py", "--db", options["db"], "--host", options["host"], "--platform", options["platform"], "--branch", options["branch"], - "--sha1", actual_snapshot, "--noprogress", "true"] + "--noprogress", "true"] if "dbhost" in options: cmd += ["--dbhost", options["dbhost"]] if "dbport" in options: @@ -339,7 +278,7 @@ def execComputeRankings(options, new_context): sys.stdout.write(" stdout: >" + stdout.strip() + "<\n") sys.stdout.write(" stderr: >" + stderr.strip() + "<\n") else: - sys.stdout.write("computerankings.py executed successfully:\n") + sys.stdout.write("updatechanges.py executed successfully:\n") sys.stdout.write(" return code: " + str(p.returncode) + "\n") sys.stdout.write(" stdout: >" + stdout.strip() + "<\n") sys.stdout.write(" stderr: >" + stderr.strip() + "<\n") @@ -494,42 +433,21 @@ sys.stdout.write("UPLOADING RESULTS, OPTIONS: " + str(options) + "\n") sys.stdout.flush() -# Reject uploading if this context is already complete (since it could -# modify the input for (and thus invalidate) any rankings computed for -# this snapshot): +# Reject uploading if this context is already complete: if contextComplete(options): sys.stderr.write( - "this snapshot is already complete for this time series -> uploading " - "rejected!\n") - sys.exit(1) - - -# Reject uploading if rankings exist for this context, since that would -# require a recomputation of those rankings. -# -# (Note that we only need to check the current SHA-1 as long as we assume that -# results are uploaded in an order that is consistent with the order of the -# corresponding SHA-1s in the branch in question: Rankings for earlier SHA-1s -# will not be affected by results for this SHA-1, and rankings for later -# SHA-1s cannot exist a this point (by assumption).) -if rankingsExist(options): - sys.stderr.write( - "error: rankings have already been computed for this context\n") + "this snapshot is already complete -> uploading rejected!\n") sys.exit(1) -# If this is the first set of results for the current context, we compute -# rankings for the same context, only one snapshot back: +# If this is the first set of results for the current context, we update +# changes for all time series in this host/platform/branch combination: if not contextExists(options): - sys.stdout.write( - "new snapshot in this time series -> attempt to compute rankings " - "for the previous snapshot ...\n") + sys.stdout.write("update changes (before registering new snapshot) ...\n") sys.stdout.flush() - execComputeRankings(options, True) + execUpdateChanges(options) else: - sys.stdout.write( - "results for this snapshot have already been uploaded -> don't " - "attempt to compute rankings at this point\n") + sys.stdout.write("skipping update changes (1)\n") sys.stdout.flush() @@ -549,22 +467,20 @@ sys.stdout.write("done\n") sys.stdout.flush() -# If no more results are expected in this context, we can compute rankings +# If no more results are expected in this context, we can update changes # already at this point. (Note: In the case that one or more uploads failed # for this context, it will be regarded as incomplete forever. In that case, -# computation of rankings for this context will be instead be triggered right +# update of changes for this context will be instead be triggered right # before uploading the first set of results for the next context (see code # above).) if contextComplete(options): sys.stdout.write( - "this snapshot is complete for this time series -> attempt to " - "compute rankings for this snapshot ...\n") + "update changes (after completion of current snapshot) ...\n") sys.stdout.flush() - execComputeRankings(options, False) + execUpdateChanges(options) else: - sys.stdout.write( - "this snapshot is incomplete for this time series -> " - "don't attempt to compute rankings at this point\n") + sys.stdout.write("skipping update changes (2)\n") + sys.stdout.flush() sys.stdout.write("UPLOADING RESULTS DONE\n") |