Improved the finalizeresults.py script.

2 files changed, 242 insertions, 149 deletions

diff --git a/scripts/finalizeresults.py b/scripts/finalizeresults.py
index 1ba53f2..0622387 100755
--- a/scripts/finalizeresults.py
+++ b/scripts/finalizeresults.py
@@ -4,10 +4,11 @@ import sys
 from xml.dom.minidom import parse
 from dbaccess import setDatabase, execQuery, commit
 from misc import (
-    textToId, getAllSnapshots, isValidSHA1, getBMTimeSeriesStatsList)
+    textToId, getAllSnapshots, getLastRankingSnapshot, isValidSHA1,
+    getBMTimeSeriesStatsList)
 
 
-# === Global functions ===============================================
+# --- BEGIN global functions ------------------------------------------------
 
 def printUsage():
     print (
@@ -30,88 +31,11 @@ def printVerboseUsage():
         "intended for experimentation).")
 
 
-# === Main program ===================================================
-
-# --- Validate arguments -------------------------------------
-
-if ((len(sys.argv) > 1) and (sys.argv[1] == "--help")):
-    printVerboseUsage()
-    sys.exit(1)
-
-if (len(sys.argv) != 6):
-    printUsage()
-    sys.exit(1)
-
-host      = sys.argv[1]
-platform  = sys.argv[2]
-branch    = sys.argv[3]
-sha1      = sys.argv[4]
-if (not isValidSHA1(sha1)):
-    print "invalid SHA-1:", sha1
-    sys.exit(1)
-db = sys.argv[5]
-
-setDatabase(db)
-
-host_id = textToId("host", host)
-if host_id == -1:
-    print "no such host:", host
-    sys.exit(1)
-platform_id = textToId("platform", platform)
-if platform_id == -1:
-    print "no such platform:", platform
-    sys.exit(1)
-branch_id = textToId("branch", branch)
-if branch_id == -1:
-    print "no such branch:", branch
-    sys.exit(1)
-sha1_id = textToId("sha1", sha1)
-if sha1_id == -1:
-    print "no such SHA-1:", sha1
-    sys.exit(1)
-
-# --- Compute time series stats ---------------------------------------
-
-# For simplicity we hardcode the tolerances for now:
-difftol = 1.1
-durtolmin = 3
-durtolmax = 30
-
-# Determine the target snapshot range:
-# (The range should end at the snapshot given on the command-line and begin
-# at the snapshot that is 2 * durtolmax snapshots back in time, or, of no such
-# snapshot exist, the first available snapshot.)
-sys.stdout.write("getting snapshots ... ")
-sys.stdout.flush()
-snapshots = getAllSnapshots(host_id, platform_id, branch_id)
-sys.stdout.write("done\n")
-sys.stdout.flush()
-try:
-    sha12_pos = zip(*snapshots)[0].index(sha1_id)
-except ValueError:
-    print "no observations found for SHA-1:", sha1
-    sys.exit(1)
-sha11_pos = max(0, (sha12_pos - 2 * durtolmax) + 1)
-snapshots = snapshots[sha11_pos:(sha12_pos + 1)]
-if len(snapshots) < 2:
-    print "no observations found before SHA-1:", sha1
-    sys.exit(1)
-
-
-# *** Get time series statistics for all benchmarks *********************
-
+# ### 2 B DOCUMENTED!
 def printProgress(p, lead):
     sys.stdout.write(lead + " ... (" + "{0:.2f}".format(p) + " %)\r")
     sys.stdout.flush()
 
-bmstats_list = getBMTimeSeriesStatsList(
-    host_id, platform_id, branch_id, snapshots, None, difftol, durtolmin,
-    durtolmax, printProgress, "getting time series statistics")
-
-sys.stdout.write("\n")
-
-
-# *** Compute rankings **************************************************
 
 # ### 2 B DOCUMENTED!
 # NOTE: This function is currently duplicated elsewhere in JavaScipt!
@@ -120,80 +44,25 @@ def changeMagnitudeScore(change):
     abs_change = (1.0 / change) if change < 1 else change
     return (min(abs_change, max_change) - 1.0) / (max_change - 1.0)
 
+
 # ### 2 B DOCUMENTED!
 # NOTE: This function is currently duplicated elsewhere in JavaScript!
 def qualityScore(lsd, ni, nz, nc, mdrse):
     max_bad_snapshots = 30 # experimental; maybe use max durability score?
     max_sample_size = 5;
-    max_LSD = max_ad_napshots;
+    max_LSD = max_bad_snapshots;
     max_NI = max_bad_snapshots * max_sample_size;
     max_NZ = max_bad_snapshots * max_sample_size;
     max_NC = max_bad_snapshots;
 
-    lsd_score = Math.min(1, lsd / float(max_LSD));
-    ni_score = Math.min(1, ni / float(max_NI));
-    var nz_score = Math.min(1, nz / float(max_NZ));
-    var nc_score = Math.min(1, nc / float(max_NC));
-    var mdrse_score = mdrse / 100.0;
+    lsd_score = min(1, lsd / float(max_LSD));
+    ni_score = min(1, ni / float(max_NI));
+    nz_score = min(1, nz / float(max_NZ));
+    nc_score = min(1, nc / float(max_NC));
+    mdrse_score = mdrse / 100.0;
 
     return (lsd_score + ni_score + nz_score + nc_score + mdrse_score) / 5.0;
-}
-
-
-sys.stdout.write("creating table for all ranking stats ... ")
-sys.stdout.flush()
-
-# Step 1: Create a table containing all ranking statistics (one row per
-#         benchmark/metric):
-table = []
-for stats in bmstats_list:
-
-    # NOTE:
-    # - All of the ranking statisticx1s are of type "higher is better"
-    #   (a high value is ranked better than a low value).
-    # - Moreover, all present/defined values are non-negative.
-    # - This means that representing absent/undefined values as -1 is ok,
-    #   since this ensures lowest ranking.
-
-    benchmark_id = stats["benchmark_id"]
-    metric_id = stats["metric_id"]
-    lsd = stats["lsd"]
-    ni = stats["ni"]
-    nz = stats["nz"]
-    nc = stats["nc"]
-    mdrse = stats["med_of_rses"]
-    rsemd = stats["rse_of_meds"]
-
-    qs = qualityScore(lsd, ni, nz, nc, mdrse)
-
-    lc = stats["lc"]
-    if lc >= 0.0:
-        lcgss = stats["lc_gsep_score"]
-        lclss = stats["lc_lsep_score"]
-        lcds1 = stats["lc_dur1_score"]
-        lcds2 = stats["lc_dur2_score"]
-        lcms = changeMagnitudeScore(lc)
-        lcss1 = lcms * lcgss * lclss * lcds1
-        lcss = lcss1 * lcds2
-        if lc < 1.0:
-            lcssr = lcss
-            lcss1r = lcss
-            lcssi = lcss1i = -1
-        else:
-            lcssi = lcss
-            lcss1i = lcss
-            lcssr = lcss1r = -1
-    else:
-        lcssr = lcssi = lcss1r = lcss1i = -1
-
-    table.append((benchmark_id, metric_id, qs, lcssr, lcssi, lcss1r, lcss1i))
 
-sys.stdout.write("done\n")
-sys.stdout.flush()
-
-
-# Step 2: Sort the table individually for each ranking statistic and register
-# the ranking positions in the database:
 
 # Registers the ranking for a given statistic.
 # Assumptions:
@@ -248,18 +117,217 @@ def registerRanking(
         execQuery(query, False)
 
 
-nameToIndex = { "QS": 2, "LCSSR": 3, "LCSSI": 4, "LCSS1R": 5, "LCSS1I": 6 }
-for name in nameToIndex:
-    sys.stdout.write("registering ranking for " + name + " ...\r")
+# ### 2 B DOCUMENTED!
+def getAllRankingStats(bmstats_list):
+    table = []
+    for stats in bmstats_list:
+
+        # NOTE:
+        # - All of the ranking statistics are of type "higher is better"
+        #   (a high value is ranked better than a low value).
+        # - Moreover, all present/defined values are non-negative.
+        # - This means that representing absent/undefined values as -1 is ok,
+        #   since this ensures lowest ranking.
+
+        benchmark_id = stats["benchmark_id"]
+        metric_id = stats["metric_id"]
+        lsd = stats["lsd"]
+        ni = stats["ni"]
+        nz = stats["nz"]
+        nc = stats["nc"]
+        mdrse = stats["med_of_rses"]
+        rsemd = stats["rse_of_meds"]
+
+        qs = qualityScore(lsd, ni, nz, nc, mdrse)
+
+        lc = stats["lc"]
+        if lc >= 0.0:
+            lcgss = stats["lc_gsep_score"]
+            lclss = stats["lc_lsep_score"]
+            lcds1 = stats["lc_dur1_score"]
+            lcds2 = stats["lc_dur2_score"]
+            lcms = changeMagnitudeScore(lc)
+            lcss1 = lcms * lcgss * lclss * lcds1
+            lcss = lcss1 * lcds2
+            if lc < 1.0:
+                lcssr = lcss
+                lcss1r = lcss
+                lcssi = lcss1i = -1
+            else:
+                lcssi = lcss
+                lcss1i = lcss
+                lcssr = lcss1r = -1
+        else:
+            lcssr = lcssi = lcss1r = lcss1i = -1
+
+        table.append(
+            (benchmark_id, metric_id, qs, lcssr, lcssi, lcss1r, lcss1i))
+
+    return table
+
+
+# ### 2 B DOCUMENTED!
+def getFirstUploadTimestamp(snapshots, sha1_id):
+    try:
+        return snapshots[zip(*snapshots)[0].index(sha1_id)][1]
+    except ValueError:
+        return -1
+
+
+# ### 2 B DOCUMENTED!
+def updateRankings(host_id, platform_id, branch_id, sha1_id):
+
+    # Get all snapshots matching the host/platform/branch combination:
+    sys.stdout.write("getting snapshots ... ")
+    sys.stdout.flush()
+    snapshots = getAllSnapshots(host_id, platform_id, branch_id)
+    sys.stdout.write("done\n")
     sys.stdout.flush()
-    registerRanking(
-        table, nameToIndex[name], name, host_id, platform_id, branch_id,
-        sha1_id)
 
-sys.stdout.write("\n")
+
+    # Rankings will normally be computed once a week for each
+    # host/context/branch combination (note the tradeoff between update
+    # frequency and database space consumption):
+    ranking_interval = 3600 * 24 * 7 # secs in a week
+
+    # Rankings will be updated if at least one of the following
+    # conditions eventually becomes True:
+    force_cond = empty_cond = interval_cond = False
+
+    force_ranking = False # Used for testing
+
+    force_cond = force_ranking
+
+    if not force_cond:
+        last_ranking_sha1_id = getLastRankingSnapshot(
+            host_id, platform_id, branch_id, snapshots)[0]
+        print "last_ranking_sha1_id:", last_ranking_sha1_id
+        empty_cond = last_ranking_sha1_id < 0
+        if not empty_cond:
+            last_ranking_timestamp = getFirstUploadTimestamp(
+                snapshots, last_ranking_sha1_id)
+            if last_ranking_timestamp < 0:
+                print (
+                "failed to extract last_ranking_timestamp (programming error?)")
+                sys.exit(1)
+            target_timestamp = getFirstUploadTimestamp(snapshots, sha1_id)
+            if target_timestamp < 0:
+                print (
+                "failed to extract target_timestamp (error in command-line " +
+                "args?)")
+                sys.exit(1)
+            interval_cond = (
+                (target_timestamp - last_ranking_timestamp) > ranking_interval)
+
+    if not (force_cond or empty_cond or interval_cond):
+        print (
+            "not updating rankings ('force', 'empty', and 'interval' " +
+            "conditions all failed)")
+        return
+
+    print (
+        "updating rankings ('force' cond.: " + str(force_cond) +
+        "; 'empty' cond.: " + str(empty_cond) +
+        "; 'interval' cond.: " + str(interval_cond) + ") ...")
+
+    # For simplicity we hardcode the tolerances for now:
+    difftol = 1.1
+    durtolmin = 3
+    durtolmax = 30
+
+    # Determine the target snapshot range:
+    # (The range should end at the snapshot given on the command-line and begin
+    # at the snapshot that is 2 * durtolmax snapshots back in time, or, if no
+    # such snapshot exists, the first available snapshot.)
+    try:
+        sha12_pos = zip(*snapshots)[0].index(sha1_id)
+    except ValueError:
+        print "no observations found for SHA-1:", sha1
+        sys.exit(1)
+    sha11_pos = max(0, (sha12_pos - 2 * durtolmax) + 1)
+    snapshots = snapshots[sha11_pos:(sha12_pos + 1)]
+    if len(snapshots) < 2:
+        print "no observations found before SHA-1:", sha1
+        sys.exit(1)
+
+    # Get time series statistics for all benchmarks:
+    bmstats_list = getBMTimeSeriesStatsList(
+        host_id, platform_id, branch_id, snapshots, None, difftol, durtolmin,
+        durtolmax, printProgress, "getting time series statistics")
+
+    sys.stdout.write("\n")
+
+
+    # *** Compute rankings **************************************************
+
+    # Step 1: Create a table containing all ranking statistics (one row per
+    #         benchmark/metric):
+    sys.stdout.write("creating table for all ranking stats ... ")
+    sys.stdout.flush()
+    table = getAllRankingStats(bmstats_list)
+    sys.stdout.write("done\n")
+    sys.stdout.flush()
+
+    # Step 2: Sort the table individually for each ranking statistic and
+    # register the ranking positions in the database:
+    nameToIndex = { "QS": 2, "LCSSR": 3, "LCSSI": 4, "LCSS1R": 5, "LCSS1I": 6 }
+    for name in nameToIndex:
+        sys.stdout.write("registering ranking for " + name + " ...\r")
+        sys.stdout.flush()
+        registerRanking(
+            table, nameToIndex[name], name, host_id, platform_id, branch_id,
+            sha1_id)
+
+    sys.stdout.write("\n")
+
+# --- END global functions ------------------------------------------------
+
+
+# --- BEGIN main program ------------------------------------------------
+
+if ((len(sys.argv) > 1) and (sys.argv[1] == "--help")):
+    printVerboseUsage()
+    sys.exit(1)
+
+if (len(sys.argv) != 6):
+    printUsage()
+    sys.exit(1)
+
+host      = sys.argv[1]
+platform  = sys.argv[2]
+branch    = sys.argv[3]
+sha1      = sys.argv[4]
+if (not isValidSHA1(sha1)):
+    print "invalid SHA-1:", sha1
+    sys.exit(1)
+db = sys.argv[5]
+
+setDatabase(db)
+
+host_id = textToId("host", host)
+if host_id == -1:
+    print "no such host:", host
+    sys.exit(1)
+platform_id = textToId("platform", platform)
+if platform_id == -1:
+    print "no such platform:", platform
+    sys.exit(1)
+branch_id = textToId("branch", branch)
+if branch_id == -1:
+    print "no such branch:", branch
+    sys.exit(1)
+sha1_id = textToId("sha1", sha1)
+if sha1_id == -1:
+    print "no such SHA-1:", sha1
+    sys.exit(1)
+
+
+updateRankings(host_id, platform_id, branch_id, sha1_id)
 
 
 # Make sure everything is written to the database:
 commit()
 
 print "finalization done"
+
+# --- END main program ------------------------------------------------
diff --git a/scripts/misc.py b/scripts/misc.py
index 1cb6a91..5d01974 100644
--- a/scripts/misc.py
+++ b/scripts/misc.py
@@ -134,6 +134,31 @@ def getAllSnapshots(host_id, platform_id, branch_id, reverse = False):
     return tuple(snapshots)
 
 
+# Gets the (SHA-1 ID, timestamp) pair associated with the most recent rankings
+# computed for the given host/platform/branch combination.
+# The 'snapshots' argument is a chronologically ordered n-tuple of
+# (SHA-1 ID, first upload timestamp) pairs. Returns (-1, -1) if no match
+# is found.
+def getLastRankingSnapshot(host_id, platform_id, branch_id, snapshots):
+
+    # Get all SHA-1 IDs in 'ranking' for this host/platform/branch
+    # combination:
+    ranking_sha1_ids = zip(*execQuery(
+        "SELECT DISTINCT sha1Id FROM ranking " +
+        "WHERE hostId = " + str(host_id) +
+        " AND platformId = " + str(platform_id) +
+        " AND branchId = " + str(branch_id) +
+        ";"))[0]
+
+    # Find the most recent one among the ranking SHA-1 IDs:
+    for sha1_id, timestamp in reversed(snapshots):
+        print "sha1_id:", sha1_id
+        if sha1_id in ranking_sha1_ids:
+            return (sha1_id, timestamp)
+
+    return (-1, -1) # not found
+
+
 # Retrieves the time series of valid median results for the given
 # benchmark/metric combination. Only the part of the time series that
 # is within the selected snapshot interval is considered.