path: root/scripts/finalizeresults.py

#!/usr/bin/env python

import sys
from xml.dom.minidom import parse
from dbaccess import setDatabase, execQuery, commit
from misc import (
    textToId, getAllSnapshots, isValidSHA1, getBMTimeSeriesStatsList)


# === Global functions ===============================================

def printUsage():
    print (
        "usage:" + sys.argv[0] +
        " --help | <host> <platform> <branch> <SHA-1> <database>")

def printVerboseUsage():
    printUsage()
    print ("<host>:      The physical machine on which the results were " +
           "produced (e.g. barbarella or 172.24.90.79).")
    print ("<platform>:  The OS/compiler/architecture combination " +
           "(e.g. linux-g++-32).")
    print ("<branch>:    The product branch (e.g. 'qt 4.6', 'qt 4.7', or " +
           "'qt master').")
    print ("<SHA-1>:     The tested revision within the branch. Can be " +
           "extracted using 'git log -1 --pretty=format:%H' (assuming the " +
           "tested revision is the current head revision).")
    print (
        "<database>:  The database. One of 'bm' or 'bm-dev' (the latter " +
        "intended for experimentation).")


# === Main program ===================================================

# --- Validate arguments -------------------------------------

if ((len(sys.argv) > 1) and (sys.argv[1] == "--help")):
    printVerboseUsage()
    sys.exit(1)

if (len(sys.argv) != 6):
    printUsage()
    sys.exit(1)

host      = sys.argv[1]
platform  = sys.argv[2]
branch    = sys.argv[3]
sha1      = sys.argv[4]
if (not isValidSHA1(sha1)):
    print "invalid SHA-1:", sha1
    sys.exit(1)
db = sys.argv[5]

setDatabase(db)

host_id = textToId("host", host)
if host_id == -1:
    print "no such host:", host
    sys.exit(1)
platform_id = textToId("platform", platform)
if platform_id == -1:
    print "no such platform:", platform
    sys.exit(1)
branch_id = textToId("branch", branch)
if branch_id == -1:
    print "no such branch:", branch
    sys.exit(1)
sha1_id = textToId("sha1", sha1)
if sha1_id == -1:
    print "no such SHA-1:", sha1
    sys.exit(1)

# --- Compute time series stats ---------------------------------------

# For simplicity we hardcode the tolerances for now:
difftol = 1.1
durtolmin = 3
durtolmax = 30

# Determine the target snapshot range:
# (The range should end at the snapshot given on the command-line and begin
# at the snapshot that is 2 * durtolmax snapshots back in time, or, of no such
# snapshot exist, the first available snapshot.)
sys.stdout.write("getting snapshots ... ")
sys.stdout.flush()
snapshots = getAllSnapshots(host_id, platform_id, branch_id)
sys.stdout.write("done\n")
sys.stdout.flush()
try:
    sha12_pos = zip(*snapshots)[0].index(sha1_id)
except ValueError:
    print "no observations found for SHA-1:", sha1
    sys.exit(1)
sha11_pos = max(0, (sha12_pos - 2 * durtolmax) + 1)
snapshots = snapshots[sha11_pos:(sha12_pos + 1)]
if len(snapshots) < 2:
    print "no observations found before SHA-1:", sha1
    sys.exit(1)


# *** Get time series statistics for all benchmarks *********************

def printProgress(p, lead):
    sys.stdout.write(lead + " ... (" + "{0:.2f}".format(p) + " %)\r")
    sys.stdout.flush()

bmstats_list = getBMTimeSeriesStatsList(
    host_id, platform_id, branch_id, snapshots, None, difftol, durtolmin,
    durtolmax, printProgress, "getting time series statistics")

sys.stdout.write("\n")


# *** Compute rankings **************************************************

def changeMagnitudeScore(change):
    max_change = 2.0
    abs_change = (1.0 / change) if change < 1 else change
    return (min(abs_change, max_change) - 1.0) / (max_change - 1.0)


sys.stdout.write("creating table for all ranking stats ... ")
sys.stdout.flush()

# Step 1: Create a table containing all ranking statistics (one row per
#         benchmark/metric):
table = []
for stats in bmstats_list:

    # NOTE:
    # - All of the ranking statisticx1s are of type "higher is better"
    #   (a high value is ranked better than a low value).
    # - Moreover, all present/defined values are non-negative.
    # - This means that representing absent/undefined values as -1 is ok,
    #   since this ensures lowest ranking.

    benchmark_id = stats["benchmark_id"]
    metric_id = stats["metric_id"]
    lsd = stats["lsd"]
    ni = stats["ni"]
    nz = stats["nz"]
    nc = stats["nc"]
    mdrse = stats["med_of_rses"]
    rsemd = stats["rse_of_meds"]

    lc = stats["lc"]
    if lc >= 0.0:
        lcgss = stats["lc_gsep_score"]
        lclss = stats["lc_lsep_score"]
        lcds1 = stats["lc_dur1_score"]
        lcds2 = stats["lc_dur2_score"]
        lcms = changeMagnitudeScore(lc)
        lcss1 = lcms * lcgss * lclss * lcds1
        lcss = lcss1 * lcds2
        if lc < 1.0:
            lcssr = lcss
            lcss1r = lcss
            lcssi = lcss1i = -1
        else:
            lcssi = lcss
            lcss1i = lcss
            lcssr = lcss1r = -1
    else:
        lcssr = lcssi = lcss1r = lcss1i = -1

    table.append((
            benchmark_id, metric_id, lsd, ni, nz, nc, mdrse, rsemd,
            lcssr, lcssi, lcss1r, lcss1i))

sys.stdout.write("done\n")
sys.stdout.flush()


# Step 2: Sort the table individually for each ranking statistic and register
# the ranking positions in the database:

# Registers the ranking for a given statistic.
# Assumptions:
# - A high value should be ranked above a small one.
# - A negative value is undefined and gets an invalid ranking position, i.e. -1.
def registerRanking(
    table, stat_index, stat_name, host_id, platform_id, branch_id, sha1_id):

    table.sort(key=lambda x: x[stat_index], reverse=True)

    stat_id = textToId("rankingStat", stat_name)
    assert stat_id >= 0

    row_pos = 0
    ranking_pos = 0
    for row in table:
        benchmark_id = row[0]
        metric_id = row[1]
        stat_value = row[stat_index]

        # The following statement ensures the following conditions:
        # - A negative value gets an invalid ranking position, i.e. -1
        # - Equal values get the same ranking position.
        # - The ranking position of benchmark B indicates the number of
        #   benchmarks ranked higher than B (i.e. having a smaller ranking
        #   position).
        if stat_value < 0:
            ranking_pos = -1
            # Note that the remaining values will now be negative, so updating
            # row_pos and prev_stat_value is no longer necessary!
        else:
            if (row_pos > 0) and (stat_value != prev_stat_value):
                ranking_pos = row_pos
            row_pos = row_pos + 1
            prev_stat_value = stat_value

        # Insert or update the corresponding row in the 'ranking' table:
        query = (
            "SELECT merge_ranking("
            + str(host_id)
            + ", " + str(platform_id)
            + ", " + str(branch_id)
            + ", " + str(sha1_id)
            + ", " + str(benchmark_id)
            + ", " + str(metric_id)
            + ", " + str(stat_id)
            + ", " + str(stat_value)
            + ", " + str(ranking_pos)
            + ");"
            )

        execQuery(query, False)


nameToIndex = {
    "LSD": 2,
    "NI": 3,
    "NZ": 4,
    "NC": 5,
    "MDRSE": 6,
    "RSEMD": 7,
    "LCSSR": 8,
    "LCSSI": 9,
    "LCSS1R": 10,
    "LCSS1I": 11
}

for name in nameToIndex:
    sys.stdout.write("registering ranking for " + name + " ...\r")
    sys.stdout.flush()
    registerRanking(
        table, nameToIndex[name], name, host_id, platform_id, branch_id,
        sha1_id)

sys.stdout.write("\n")


# Make sure everything is written to the database:
commit()

print "finalization done"