path: root/scripts/uploadresults.py

#!/usr/bin/env python

import sys
from xml.dom.minidom import parse
from dbaccess import setDatabase, execQuery, commit
from misc import getOptions, textToId, isValidSHA1, getContext


# --- BEGIN Global functions ----------------------------------------------

def printUsage():
    print (
        "usage:" + sys.argv[0] +
        " --help | [--dbhost H --dbport P] --db D --host H --platform P "
        "--branch B --sha1 S --file F")

def printVerboseUsage():
    printUsage()
    print "\noptions:"
    print(
        "    --help: This help.")
    print(
        "  --dbhost: The database server host (overriding the default).")
    print(
        "  --dbport: The database server port (overriding the default).")
    print(
        "      --db: The database. One of 'bm' or 'bm-dev' (the latter "
        "intended for experimentation).")
    print(
        "    --host: The physical machine on which the results were "
        "produced (e.g. barbarella or 172.24.90.79).")
    print(
        "--platform: The OS/compiler/architecture combination "
        "(e.g. linux-g++-32).")
    print(
        "  --branch: The product branch (e.g. 'qt 4.6', 'qt 4.7', or "
        "'qt master').")
    print(
        "    --sha1: The tested revision within the branch. Can be "
        "extracted using 'git log -1 --pretty=format:%H' (assuming the "
        "tested revision is the current head revision).")
    print(
        "    --file: The results file in QTestLib XML output format.")

# Returns True iff a low value indicates better performance than a high
# value for the given metric.
def lowerIsBetter(metric):
    return {
        "walltimemilliseconds": True,
        "walltime": True,
        "cputicks": True,
        "instructionreads": True,
        "events": True,
        "bitspersecond": False,
        "bytespersecond": False,
        "framespersecond": False,
        "fps": False
        # add more if necessary ...
        }[metric.lower()]

# Returns the canonical (i.e. "unaliased") form of the metric name.
def canonicalMetric(metric):
    if (metric.lower() == "walltime"):
        return "WalltimeMilliseconds"
    if (metric.lower() == "framespersecond"):
        return "fps"
    return metric

# Returns True iff at least one of the given incidents indicates failure for
# the given data tag.
def matchesFailedIncident(dataTag, incidents):
    for incident in incidents:
        if (incident.getAttribute("type") == "fail"):
            try:
                dataTagElem = incident.getElementsByTagName("DataTag")[0]
            except:
                continue
            if (dataTagElem.childNodes[0].data == dataTag):
                return True
    return False

# Returns results extracted from a file in QTestLib XML format
# (note: multiple top-level TestCase elements are allowed).
def extractResults(file):

    def processBenchmarkResults(
        results, dom, testCase, testFuncElem, testFunction, incidents):
        # Loop over benchmark results ...
        bmResultElems = testFuncElem.getElementsByTagName("BenchmarkResult")
        for bmResultElem in bmResultElems:

            # Data tag (note that "" is a valid data tag):
            dataTag = bmResultElem.getAttribute("tag").strip()

            # Metric:
            metric = bmResultElem.getAttribute("metric").strip()
            try:
                lowerIsBetter_ = lowerIsBetter(metric)
            except:
                print(
                    "WARNING: skipping result for unsupported metric: >" +
                    metric + "<")
                continue
            metric = canonicalMetric(metric)

            # Value:
            value = float(bmResultElem.getAttribute("value"))

            # Iterations (optional):
            iterAttr = bmResultElem.getAttribute("iterations").strip()
            if (iterAttr != ""):
                try:
                    iterations = int(iterAttr)
                    assert iterations > 0
                    value = value / iterations
                except:
                    raise BaseException(
                        "found 'iterations' attribute that is not a " +
                        "positive integer: " + iterAttr)

            # Valid:
            valid = not matchesFailedIncident(dataTag, incidents)

            # Add item to array ...
            results.append(
                {'testCase': testCase,
                 'testFunction': testFunction,
                 'dataTag': dataTag,
                 'metric': metric,
                 'lowerIsBetter': lowerIsBetter_,
                 'value': value,
                 'valid': valid})

    def processTestFunctions(results, dom, testCaseElem, testCase):
        # Loop over test functions ...
        testFuncElems = testCaseElem.getElementsByTagName("TestFunction")
        for testFuncElem in testFuncElems:
            testFunction = testFuncElem.getAttribute("name").strip()
            assert testFunction != ""
            incidents = testFuncElem.getElementsByTagName("Incident")
            processBenchmarkResults(
                results, dom, testCase, testFuncElem,
                testFunction, incidents)

    def processTestCases(results, dom):
        # Loop over test cases ...
        testCaseElems = dom.getElementsByTagName("TestCase")
        for testCaseElem in testCaseElems:
            testCase = testCaseElem.getAttribute("name").strip()
            assert testCase != ""
            processTestFunctions(
                results, dom, testCaseElem, testCase)

    # Load DOM structure from file:
    try:
        dom = parse(file)
    except:
        raise BaseException(sys.exc_info())

    # Extract benchmark results from DOM structure:
    results = []
    processTestCases(results, dom)
    dom.unlink()

    return results

# ### 2 B DOCUMENTED!
def findOrInsertId(table, value, *args):

    #print "value: >" + value + "<, ",
    query_result = execQuery(
        "SELECT id FROM " + table + " WHERE value = '" + str(value) + "';")
    if len(query_result) == 1:
        # Found, so return ID:
        #print "returning existing ID: >" + str(query_result[0][0]) + "<"
        return query_result[0][0]

    # Not found, so insert:
    query = "INSERT INTO " + table + " (value"
    for i in range(0, len(args), 2):
        query += ", " + str(args[i])
    query += ") VALUES ('" + str(value) + "'"
    for i in range(0, len(args), 2):
        query += ", " + str(args[i + 1])

    # ... and retrieve ID:
    query += ") RETURNING id;"
    query_result = execQuery(query)

    assert len(query_result) == 1
    #print "returning new ID: >" + str(query_result[0][0]) + "<"
    return query_result[0][0]


# Uploads a set of results to the database.
def uploadToDatabase(host, platform, branch, sha1, results):

    # Append a row to the 'upload' table (to record this upload event) ...
    execQuery("INSERT INTO upload DEFAULT VALUES;", False)

    # Retrieve the ID of the row we just inserted ...
    uploadId = execQuery("SELECT currval('upload_id_seq');")[0][0]

    hostId = findOrInsertId("host", host)
    platformId = findOrInsertId("platform", platform)
    branchId = findOrInsertId("branch", branch)
    sha1Id = findOrInsertId("sha1", sha1)

    contextId = getContext(hostId, platformId, branchId, sha1Id)
    if contextId == -1:
        contextId = execQuery(
            "INSERT INTO context"
            " (hostId, platformId, branchId, sha1Id)"
            " VALUES (%d, %d, %d, %d)"
            " RETURNING id;"
            % (hostId, platformId, branchId, sha1Id))[0][0]

    # Append rows to the 'result' table ...
    for result in results:
        benchmark = (
            result['testCase'] + ":" + result['testFunction'] + "(" +
            str(result['dataTag']) + ")")
        benchmarkId = findOrInsertId("benchmark", benchmark)

        metricId = findOrInsertId(
            "metric", result['metric'], "lowerIsBetter",
            result['lowerIsBetter'])

        query = (
            "INSERT INTO result"
            " (contextId, benchmarkId, value, valid, metricId, uploadId)"
            " VALUES (%d, %d, %f, %s, %d, %d);"
            % (contextId, benchmarkId, result['value'], result['valid'],
               metricId, uploadId))

        execQuery(query, False)

    # Write to database:
    commit()


# Returns True iff rankings exist for the given context.
def rankingsExist(host, platform, branch, sha1):
    context_id = getContext(
        textToId('host', host),
        textToId('platform', platform),
        textToId('branch', branch),
        textToId('sha1', sha1))

    matches = execQuery(
        "SELECT id FROM ranking WHERE context2Id = %d LIMIT 1;" % context_id)

    return len(matches) > 0


# --- END Global functions ----------------------------------------------


# --- BEGIN Main program ----------------------------------------------

options, http_get = getOptions()

if "help" in options:
    printVerboseUsage()
    sys.exit(0)

if (not ("db" in options and "host" in options and "platform" in options and
    "branch" in options and "sha1" in options and "file" in options)):
    printUsage()
    sys.exit(0)

if not isValidSHA1(options["sha1"]):
    print "error: invalid SHA-1:", options["sha1"]
    sys.exit(1)

setDatabase(
    options["dbhost"] if "dbhost" in options else None,
    options["dbport"] if "dbport" in options else None,
    options["db"])

# Reject uploading if rankings exist for this context, since that would
# require a recomputation of the rankings.
#
# (Note that this check if sufficient as long as we assume that results
# are uploaded in an order that is consistent with the order of the
# corresponding SHA-1s in the branch in question (i.e. that results will
# never be uploaded for a SHA-1 that precedes (is "older" than) a SHA-1 for
# which results already exist). This assumption ensures that results for a
# new SHA-1 will not alter the input for any existing ranking.)
if rankingsExist(
    options["host"], options["platform"], options["branch"], options["sha1"]):
    print "error: this context has already been finalized"
    sys.exit(1)

try:
    results = extractResults(options["file"])
except BaseException as e:
    print "error: failed to parse XML file:", e.args[0]
    sys.exit(1)

uploadToDatabase(
    options["host"], options["platform"], options["branch"], options["sha1"],
    results)

print "uploading done"

# --- END Main program ----------------------------------------------