path: root/scripts/uploadresults.py

#!/usr/bin/env python

import sys
from subprocess import Popen, PIPE
from xml.dom.minidom import parse
from dbaccess import setDatabase, execQuery, commit
from misc import (
    getOptions, textToId, idToText, isValidSHA1, getContext, getAllSnapshots)


# --- BEGIN Global functions ----------------------------------------------

def printUsage():
    print (
        "usage:" + sys.argv[0] +
        " --help | [--dbhost H --dbport P] --db D --host H --platform P "
        "--branch B --sha1 S --file F")

def printVerboseUsage():
    printUsage()
    print "\noptions:"
    print(
        "    --help: This help.")
    print(
        "  --dbhost: The database server host (overriding the default).")
    print(
        "  --dbport: The database server port (overriding the default).")
    print(
        "      --db: The database. One of 'bm' or 'bm-dev' (the latter "
        "intended for experimentation).")
    print(
        "    --host: The physical machine on which the results were "
        "produced (e.g. barbarella or 172.24.90.79).")
    print(
        "--platform: The OS/compiler/architecture combination "
        "(e.g. linux-g++-32).")
    print(
        "  --branch: The product branch (e.g. 'qt 4.6', 'qt 4.7', or "
        "'qt master').")
    print(
        "    --sha1: The tested revision within the branch. Can be "
        "extracted using 'git log -1 --pretty=format:%H' (assuming the "
        "tested revision is the current head revision).")
    print(
        "    --file: The results file in QTestLib XML output format.")

# Returns True iff a low value indicates better performance than a high
# value for the given metric.
def lowerIsBetter(metric):
    return {
        "walltimemilliseconds": True,
        "walltime": True,
        "cputicks": True,
        "instructionreads": True,
        "events": True,
        "bitspersecond": False,
        "bytespersecond": False,
        "framespersecond": False,
        "fps": False
        # add more if necessary ...
        }[metric.lower()]

# Returns the canonical (i.e. "unaliased") form of the metric name.
def canonicalMetric(metric):
    if (metric.lower() == "walltime"):
        return "WalltimeMilliseconds"
    if (metric.lower() == "framespersecond"):
        return "fps"
    return metric

# Returns True iff at least one of the given incidents indicates failure for
# the given data tag.
def matchesFailedIncident(dataTag, incidents):
    for incident in incidents:
        if (incident.getAttribute("type") == "fail"):
            try:
                dataTagElem = incident.getElementsByTagName("DataTag")[0]
            except:
                continue
            if (dataTagElem.childNodes[0].data == dataTag):
                return True
    return False

# Returns results extracted from a file in QTestLib XML format
# (note: multiple top-level TestCase elements are allowed).
def extractResults(file):

    def processBenchmarkResults(
        results, dom, testCase, testFuncElem, testFunction, incidents):
        # Loop over benchmark results ...
        bmResultElems = testFuncElem.getElementsByTagName("BenchmarkResult")
        for bmResultElem in bmResultElems:

            # Data tag (note that "" is a valid data tag):
            dataTag = bmResultElem.getAttribute("tag").strip()

            # Metric:
            metric = bmResultElem.getAttribute("metric").strip()
            try:
                lowerIsBetter_ = lowerIsBetter(metric)
            except:
                print(
                    "WARNING: skipping result for unsupported metric: >" +
                    metric + "<")
                continue
            metric = canonicalMetric(metric)

            # Value:
            value = float(bmResultElem.getAttribute("value"))

            # Iterations (optional):
            iterAttr = bmResultElem.getAttribute("iterations").strip()
            if (iterAttr != ""):
                try:
                    iterations = int(iterAttr)
                    assert iterations > 0
                    value = value / iterations
                except:
                    raise BaseException(
                        "found 'iterations' attribute that is not a " +
                        "positive integer: " + iterAttr)

            # Valid:
            valid = not matchesFailedIncident(dataTag, incidents)

            # Add item to array ...
            results.append(
                {'testCase': testCase,
                 'testFunction': testFunction,
                 'dataTag': dataTag,
                 'metric': metric,
                 'lowerIsBetter': lowerIsBetter_,
                 'value': value,
                 'valid': valid})

    def processTestFunctions(results, dom, testCaseElem, testCase):
        # Loop over test functions ...
        testFuncElems = testCaseElem.getElementsByTagName("TestFunction")
        for testFuncElem in testFuncElems:
            testFunction = testFuncElem.getAttribute("name").strip()
            assert testFunction != ""
            incidents = testFuncElem.getElementsByTagName("Incident")
            processBenchmarkResults(
                results, dom, testCase, testFuncElem,
                testFunction, incidents)

    def processTestCases(results, dom):
        # Loop over test cases ...
        testCaseElems = dom.getElementsByTagName("TestCase")
        for testCaseElem in testCaseElems:
            testCase = testCaseElem.getAttribute("name").strip()
            assert testCase != ""
            processTestFunctions(
                results, dom, testCaseElem, testCase)

    # Load DOM structure from file:
    try:
        dom = parse(file)
    except:
        raise BaseException(sys.exc_info())

    # Extract benchmark results from DOM structure:
    results = []
    processTestCases(results, dom)
    dom.unlink()

    return results

# ### 2 B DOCUMENTED!
def findOrInsertId(table, value, *args):

    #print "value: >" + value + "<, ",
    query_result = execQuery(
        "SELECT id FROM " + table + " WHERE value = '" + str(value) + "';")
    if len(query_result) == 1:
        # Found, so return ID:
        #print "returning existing ID: >" + str(query_result[0][0]) + "<"
        return query_result[0][0]

    # Not found, so insert:
    query = "INSERT INTO " + table + " (value"
    for i in range(0, len(args), 2):
        query += ", " + str(args[i])
    query += ") VALUES ('" + str(value) + "'"
    for i in range(0, len(args), 2):
        query += ", " + str(args[i + 1])

    # ... and retrieve ID:
    query += ") RETURNING id;"
    query_result = execQuery(query)

    assert len(query_result) == 1
    #print "returning new ID: >" + str(query_result[0][0]) + "<"
    return query_result[0][0]


# Uploads a set of results to the database.
def uploadToDatabase(host, platform, branch, sha1, results):

    # Append a row to the 'upload' table (to record this upload event) ...
    execQuery("INSERT INTO upload DEFAULT VALUES;", False)

    # Retrieve the ID of the row we just inserted ...
    uploadId = execQuery("SELECT currval('upload_id_seq');")[0][0]

    hostId = findOrInsertId("host", host)
    platformId = findOrInsertId("platform", platform)
    branchId = findOrInsertId("branch", branch)
    sha1Id = findOrInsertId("sha1", sha1)

    contextId = getContext(hostId, platformId, branchId, sha1Id)
    if contextId == -1:
        contextId = execQuery(
            "INSERT INTO context"
            " (hostId, platformId, branchId, sha1Id)"
            " VALUES (%d, %d, %d, %d)"
            " RETURNING id;"
            % (hostId, platformId, branchId, sha1Id))[0][0]

    # Append rows to the 'result' table ...
    for result in results:
        benchmark = (
            result['testCase'] + ":" + result['testFunction'] + "(" +
            str(result['dataTag']) + ")")
        benchmarkId = findOrInsertId("benchmark", benchmark)

        metricId = findOrInsertId(
            "metric", result['metric'], "lowerIsBetter",
            result['lowerIsBetter'])

        query = (
            "INSERT INTO result"
            " (contextId, benchmarkId, value, valid, metricId, uploadId)"
            " VALUES (%d, %d, %f, %s, %d, %d);"
            % (contextId, benchmarkId, result['value'], result['valid'],
               metricId, uploadId))

        execQuery(query, False)

    # Write to database:
    commit()


# Returns True iff rankings exist for the given context.
def rankingsExist(options):
    context_id = getContext(
        textToId('host', options["host"]),
        textToId('platform', options["platform"]),
        textToId('branch', options["branch"]),
        textToId('sha1', options["sha1"]))

    matches = execQuery(
        "SELECT id FROM ranking WHERE context2Id = %d LIMIT 1;" % context_id)

    return len(matches) > 0


# Returns the context ID if found, otherwise -1:
def getContextIdFromNames(options):
    host_id = textToId("host", options["host"])
    platform_id = textToId("platform", options["platform"])
    branch_id = textToId("branch", options["branch"])
    sha1_id = textToId("sha1", options["sha1"])
    return getContext(host_id, platform_id, branch_id, sha1_id)


# Returns True iff this context exists:
def contextExists(options):
    return getContextIdFromNames(options) != -1


# Returns True iff no more results are to be expected for this context:
def contextComplete(options):

    max_sample_size = 5 # WARNING: This value must match the corresponding value
                        # in the script that triggers benchmark execution.

    context_id = getContextIdFromNames(options)
    sample_size = execQuery(
        "SELECT count(*) FROM"
        " (SELECT DISTINCT uploadId from result where contextId=%d) AS foo;"
        % context_id)[0][0]

    return sample_size >= max_sample_size


# Executes the external computerankings.py script with appropriate arguments.
# If new_context is True, this snapshot is assumed to be the first one in
# this time series (host/platform/branch combination), and no results have been
# uploaded for this context yet. In that case, rankings will instead be
# computed for the latest existing snapshot in this time series.
def execComputeRankings(options, new_context):

    if new_context:
        # Attempt to use the latest available snapshot for this
        # host/platform/branch combination as the actual snapshot:
        host_id = textToId("host", options["host"])
        platform_id = textToId("platform", options["platform"])
        branch_id = textToId("branch", options["branch"])
        snapshots = getAllSnapshots(host_id, platform_id, branch_id)
        if len(snapshots) > 0:
            actual_snapshot = idToText("sha1", snapshots[-1][0])
            if actual_snapshot == options["sha1"]:
                sys.stderr.write(
                    "error: context unexpectedly exists in database\n")
                sys.exit(1)
        else:
            return # special case when no snapshots exist yet
    else:
        actual_snapshot = options["sha1"]

    cmd = [
        "computerankings.py", "--db", options["db"], "--host", options["host"],
        "--platform", options["platform"], "--branch", options["branch"],
        "--sha1", actual_snapshot, "--noprogress", "true"]
    if "dbhost" in options:
        cmd += ["--dbhost", options["dbhost"]]
    if "dbport" in options:
        cmd += ["--dbport", options["dbport"]]

    p = Popen(cmd, stdout = PIPE, stderr = PIPE)
    stdout, stderr = p.communicate()
    if (p.returncode != 0):
        print "failed to execute command '" + cmd + "':"
        print "  return code:", p.returncode
        print "  stdout: >%s<" % stdout.strip()
        print "  stderr: >%s<" % stderr.strip()
    else:
        print "computerankings.py executed successfully:"
        print "  return code:", p.returncode
        print "  stdout: >%s<" % stdout.strip()
        print "  stderr: >%s<" % stderr.strip()


# --- END Global functions ----------------------------------------------


# --- BEGIN Main program ----------------------------------------------

options, http_get = getOptions()

if "help" in options:
    printVerboseUsage()
    sys.exit(0)

if (not ("db" in options and "host" in options and "platform" in options and
    "branch" in options and "sha1" in options and "file" in options)):
    printUsage()
    sys.exit(0)

if not isValidSHA1(options["sha1"]):
    sys.stderr.write("error: invalid SHA-1: " + options["sha1"] + "\n")
    sys.exit(1)

setDatabase(
    options["dbhost"] if "dbhost" in options else None,
    options["dbport"] if "dbport" in options else None,
    options["db"])


sys.stdout.write("UPLOADING RESULTS, OPTIONS: " + str(options) + "\n")
sys.stdout.flush()


# Reject uploading if this context is already complete:
if contextComplete(options):
    sys.stderr.write(
        "this snapshot is already complete for this time series -> uploading "
        "rejected!\n")
    sys.exit(1)


# Reject uploading if rankings exist for this context, since that would
# require a recomputation of those rankings.
#
# (Note that we only need to check the current SHA-1 as long as we assume that
# results are uploaded in an order that is consistent with the order of the
# corresponding SHA-1s in the branch in question: Rankings for earlier SHA-1s
# will not be affected by results for this SHA-1, and rankings for later
# SHA-1s cannot exist a this point (by assumption).)
if rankingsExist(options):
    sys.stderr.write(
        "error: rankings have already been computed for this context\n")
    sys.exit(1)


# If this is the first set of results for the current context, we compute
# rankings for the same context, only one snapshot back:
if not contextExists(options):
    sys.stdout.write(
        "new snapshot in this time series -> attempt to compute rankings "
        "for the previous snapshot ...\n")
    sys.stdout.flush()
    execComputeRankings(options, True)
else:
    sys.stdout.write(
        "results for this snapshot have already been uploaded -> don't "
        "attempt to compute rankings at this point\n")
    sys.stdout.flush()


# Parse results and store them in the database:
sys.stdout.write("parsing results and uploading to database ... ")
sys.stdout.flush()
try:
    results = extractResults(options["file"])
except BaseException as e:
    sys.stderr.write(
        "error: failed to parse XML file: " + str(e.args[0]) + "\n")
    sys.exit(1)
uploadToDatabase(
    options["host"], options["platform"], options["branch"], options["sha1"],
    results)
sys.stdout.write("done\n")
sys.stdout.flush()


# If no more results are expected in this context, we can compute rankings
# already at this point. (Note: In the case that one or more uploads failed
# for this context, it will be regarded as incomplete forever. In that case,
# computation of rankings for this context will be instead be triggered right
# before uploading the first set of results for the next context (see code
# above).)
if contextComplete(options):
    sys.stdout.write(
        "this snapshot is complete for this time series -> attempt to "
        "compute rankings for this snapshot ...\n")
    sys.stdout.flush()
    execComputeRankings(options, False)
else:
    sys.stdout.write(
        "this snapshot is incomplete for this time series -> "
        "don't attempt to compute rankings at this point\n")


sys.stdout.write("UPLOADING RESULTS DONE\n")

# --- END Main program ----------------------------------------------