path: root/scripts/uploadresults.py

#!/usr/bin/env python

import sys
from subprocess import Popen, PIPE
from xml.dom.minidom import parse, getDOMImplementation
from dbaccess import setDatabase, execQuery, commit
from misc import (
    getOptions, textToId, idToText, isValidSHA1, getContext, getAllSnapshots)


# --- BEGIN Global functions ----------------------------------------------

def printUsage():
    sys.stderr.write(
        "usage: " + sys.argv[0] +
        " --help | [--remote R] [--dbhost H --dbport P] --db D --host H "
        "--platform P --branch B --sha1 S --file F\n")


def printVerboseUsage():
    printUsage()
    sys.stderr.write("\noptions:\n")
    sys.stderr.write(
        "    --help: This help.\n")
    sys.stderr.write(
        "  --remote: A flag (0 or 1; default = 0) indicating whether this "
        "script runs on a different server than gitestr.nokia.troll.no.\n")
    sys.stderr.write(
        "  --dbhost: The database server host (overriding the default).\n")
    sys.stderr.write(
        "  --dbport: The database server port (overriding the default).\n")
    sys.stderr.write(
        "      --db: The database. One of 'bm' or 'bm-dev' (the latter "
        "intended for experimentation).\n")
    sys.stderr.write(
        "    --host: The physical machine on which the results were "
        "produced (e.g. barbarella or 172.24.90.79).\n")
    sys.stderr.write(
        "--platform: The OS/compiler/architecture combination "
        "(e.g. linux-g++-32).\n")
    sys.stderr.write(
        "  --branch: The product branch (e.g. 'qt 4.6', 'qt 4.7', or "
        "'qt master').\n")
    sys.stderr.write(
        "    --sha1: The tested revision within the branch. Can be "
        "extracted using 'git log -1 --pretty=format:%H' (assuming the "
        "tested revision is the current head revision).\n")
    sys.stderr.write(
        "    --file: The results file in QTestLib XML output format.\n")


# Returns True iff a low value indicates better performance than a high
# value for the given metric.
def lowerIsBetter(metric):
    return {
        "walltimemilliseconds": True,
        "walltime": True,
        "cputicks": True,
        "instructionreads": True,
        "events": True,
        "bitspersecond": False,
        "bytespersecond": False,
        "framespersecond": False,
        "fps": False
        # add more if necessary ...
        }[metric.lower()]

# Returns the canonical (i.e. "unaliased") form of the metric name.
def canonicalMetric(metric):
    if (metric.lower() == "walltime"):
        return "WalltimeMilliseconds"
    if (metric.lower() == "framespersecond"):
        return "fps"
    return metric


# Returns True iff at least one of the given incidents indicates failure for
# the given data tag.
def matchesFailedIncident(dataTag, incidents):
    for incident in incidents:
        if (incident.getAttribute("type") == "fail"):

            dataTagElems = incident.getElementsByTagName("DataTag")

            if dataTag == "": # special case
                assert len(dataTagElems) == 0
                return True

            try:
                dataTagElem = dataTagElems[0]
            except:
                continue
            if (dataTagElem.childNodes[0].data == dataTag):
                return True

    return False


# Returns results extracted from a file in QTestLib XML format
# (note: multiple top-level TestCase elements are allowed).
def extractResults(file):

    def processBenchmarkResults(
        results, dom, testCase, testFuncElem, testFunction, incidents):
        # Loop over benchmark results ...
        bmResultElems = testFuncElem.getElementsByTagName("BenchmarkResult")
        for bmResultElem in bmResultElems:

            # Data tag (note that "" is a valid data tag):
            dataTag = bmResultElem.getAttribute("tag").strip()

            # Metric:
            metric = bmResultElem.getAttribute("metric").strip()
            try:
                lowerIsBetter_ = lowerIsBetter(metric)
            except:
                sys.stdout.write(
                    "WARNING: skipping result for unsupported metric: >" +
                    metric + "<\n")
                continue
            metric = canonicalMetric(metric)

            # Value:
            value = float(bmResultElem.getAttribute("value"))

            # Iterations (optional):
            iterAttr = bmResultElem.getAttribute("iterations").strip()
            if (iterAttr != ""):
                try:
                    iterations = int(iterAttr)
                    assert iterations > 0
                    value = value / iterations
                except:
                    raise BaseException(
                        "found 'iterations' attribute that is not a " +
                        "positive integer: " + iterAttr)

            # Valid:
            valid = not matchesFailedIncident(dataTag, incidents)

            # Add item to array ...
            results.append(
                {'testCase': testCase,
                 'testFunction': testFunction,
                 'dataTag': dataTag,
                 'metric': metric,
                 'lowerIsBetter': lowerIsBetter_,
                 'value': value,
                 'valid': valid})

    def processTestFunctions(results, dom, testCaseElem, testCase):
        # Loop over test functions ...
        testFuncElems = testCaseElem.getElementsByTagName("TestFunction")
        for testFuncElem in testFuncElems:
            testFunction = testFuncElem.getAttribute("name").strip()
            assert testFunction != ""
            incidents = testFuncElem.getElementsByTagName("Incident")
            processBenchmarkResults(
                results, dom, testCase, testFuncElem,
                testFunction, incidents)

    def processTestCases(results, dom):
        # Loop over test cases ...
        testCaseElems = dom.getElementsByTagName("TestCase")
        for testCaseElem in testCaseElems:
            testCase = testCaseElem.getAttribute("name").strip()
            assert testCase != ""
            processTestFunctions(
                results, dom, testCaseElem, testCase)

    # Load DOM structure from file:
    try:
        dom = parse(file)
    except:
        raise BaseException(sys.exc_info())

    # Extract benchmark results from DOM structure:
    results = []
    processTestCases(results, dom)
    dom.unlink()

    return results


# ### 2 B DOCUMENTED!
def findOrInsertId(table, value, *args):

    query_result = execQuery(
        "SELECT id FROM " + table + " WHERE value = %s", (value,))
    if len(query_result) == 1:
        # Found, so return ID:
        return query_result[0][0]

    # Not found, so insert:
    query = "INSERT INTO " + table + " (value"
    for i in range(0, len(args), 2):
        query += ", " + args[i]
    query += ") VALUES (%s"
    values = [value]
    for i in range(0, len(args), 2):
        query += ", %s"
        values.append(args[i + 1])

    # ... and retrieve ID:
    query += ") RETURNING id"
    query_result = execQuery(query, values)

    assert len(query_result) == 1
    return query_result[0][0]


# Uploads a set of results to the database.
def uploadToDatabase(host, platform, branch, sha1, results):

    # Append a row to the 'upload' table (to record this upload event) ...
    execQuery("INSERT INTO upload DEFAULT VALUES", (), False)

    # Retrieve the ID of the row we just inserted ...
    uploadId = execQuery("SELECT currval('upload_id_seq')", ())[0][0]

    hostId = findOrInsertId("host", host)
    platformId = findOrInsertId("platform", platform)
    branchId = findOrInsertId("branch", branch)
    sha1Id = findOrInsertId("sha1", sha1)

    contextId = getContext(hostId, platformId, branchId, sha1Id)
    if contextId == -1:
        contextId = execQuery(
            "INSERT INTO context"
            " (hostId, platformId, branchId, sha1Id)"
            " VALUES (%s, %s, %s, %s)"
            " RETURNING id",
            (hostId, platformId, branchId, sha1Id))[0][0]

    # Append rows to the 'result' table ...
    for result in results:
        benchmark = (
            result['testCase'] + ":" + result['testFunction'] + "(" +
            str(result['dataTag']) + ")")
        benchmarkId = findOrInsertId("benchmark", benchmark)

        metricId = findOrInsertId(
            "metric", result['metric'], "lowerIsBetter",
            result['lowerIsBetter'])

        execQuery(
            "INSERT INTO result"
            " (contextId, benchmarkId, value, valid, metricId, uploadId)"
            " VALUES (%s, %s, %s, %s, %s, %s)",
            (contextId, benchmarkId, result['value'], result['valid'],
             metricId, uploadId), False)

    # Write to database:
    commit()


# Returns True iff rankings exist for the given context.
def rankingsExist(options):
    context_id = getContext(
        textToId('host', options["host"]),
        textToId('platform', options["platform"]),
        textToId('branch', options["branch"]),
        textToId('sha1', options["sha1"]))

    matches = execQuery(
        "SELECT id FROM ranking WHERE context2Id = %s LIMIT 1", (context_id,))

    return len(matches) > 0


# Returns the context ID if found, otherwise -1:
def getContextIdFromNames(options):
    host_id = textToId("host", options["host"])
    platform_id = textToId("platform", options["platform"])
    branch_id = textToId("branch", options["branch"])
    sha1_id = textToId("sha1", options["sha1"])
    return getContext(host_id, platform_id, branch_id, sha1_id)


# Returns True iff this context exists:
def contextExists(options):
    return getContextIdFromNames(options) != -1


# Returns True iff no more results are to be expected for this context:
def contextComplete(options):

    max_sample_size = 5 # WARNING: This value must match the corresponding value
                        # in the script that triggers benchmark execution.

    context_id = getContextIdFromNames(options)
    sample_size = execQuery(
        "SELECT count(*) FROM"
        " (SELECT DISTINCT uploadId from result where contextId = %s) AS foo",
        (context_id,))[0][0]

    return sample_size >= max_sample_size


# Executes the external computerankings.py script with appropriate arguments.
# If new_context is True, this snapshot is assumed to be the first one in
# this time series (host/platform/branch combination), and no results have been
# uploaded for this context yet. In that case, rankings will instead be
# computed for the latest existing snapshot in this time series.
def execComputeRankings(options, new_context):

    if new_context:
        # Attempt to use the latest available snapshot for this
        # host/platform/branch combination as the actual snapshot:
        host_id = textToId("host", options["host"])
        platform_id = textToId("platform", options["platform"])
        branch_id = textToId("branch", options["branch"])
        snapshots = getAllSnapshots(host_id, platform_id, branch_id)
        if len(snapshots) > 0:
            actual_snapshot = idToText("sha1", snapshots[-1][0])
            if actual_snapshot == options["sha1"]:
                sys.stderr.write(
                    "error: context unexpectedly exists in database\n")
                sys.exit(1)
        else:
            return # special case when no snapshots exist yet
    else:
        actual_snapshot = options["sha1"]

    cmd = [
        "computerankings.py", "--db", options["db"], "--host", options["host"],
        "--platform", options["platform"], "--branch", options["branch"],
        "--sha1", actual_snapshot, "--noprogress", "true"]
    if "dbhost" in options:
        cmd += ["--dbhost", options["dbhost"]]
    if "dbport" in options:
        cmd += ["--dbport", options["dbport"]]

    p = Popen(cmd, stdout = PIPE, stderr = PIPE)
    stdout, stderr = p.communicate()
    if (p.returncode != 0):
        sys.stdout.write("failed to execute command '" + str(cmd) + "':\n")
        sys.stdout.write("  return code: " + str(p.returncode) + "\n")
        sys.stdout.write("  stdout: >" + stdout.strip() + "<\n")
        sys.stdout.write("  stderr: >" + stderr.strip() + "<\n")
    else:
        sys.stdout.write("computerankings.py executed successfully:\n")
        sys.stdout.write("  return code: " + str(p.returncode) + "\n")
        sys.stdout.write("  stdout: >" + stdout.strip() + "<\n")
        sys.stdout.write("  stderr: >" + stderr.strip() + "<\n")


# Executes an external command. Returns True iff the return code of the
# command is zero.
def runCommand(cmd, cwd_ = None):
    p = Popen(cmd, cwd = cwd_, stdout = PIPE, stderr = PIPE)
    stdout, stderr = p.communicate()
    if p.returncode != 0:
        print "\nfailed to run command", cmd
        print "  return code:", p.returncode
        print "  stdout: >%s<" % stdout.strip()
        print "  stderr: >%s<" % stderr.strip()
        return False
    return True


# Updates a 'testable snapshots' file on gitestr.nokia.troll.no that can be
# used by external test clients to determine the most recent snapshot (SHA-1)
# of each branch and also how many uploads have been completed by each
# host/platform combination for these branch/snapshot combinations.
#
# This script is assumed to run on a different host than
# gitestr.nokia.troll.no iff the 'remote' argument is True.
#
def updateTestableSnapshotsFile(remote):

    # --- BEGIN create XML structure ---
    impl = getDOMImplementation()
    doc = impl.createDocument(None, "branches", None)
    branches_elem = doc.documentElement

    # Loop over branches:
    branch_ids = execQuery("SELECT DISTINCT branchId FROM context", ())
    for branch_id in zip(*branch_ids)[0]:
        # Get the latest snapshot for this branch:
        sha1_id = execQuery(
            "SELECT sha1Id FROM context WHERE branchId = %s"
            " ORDER BY timestamp DESC LIMIT 1",
            (branch_id,))[0][0]

        # Create 'branch' element:
        branch_elem = doc.createElement("branch")
        branch_elem.setAttribute("name", idToText("branch", branch_id))
        branch_elem.setAttribute("sha1", idToText("sha1", sha1_id))
        branches_elem.appendChild(branch_elem)

        # Loop over contexts matching this branch/snapshot combination:
        contexts = execQuery(
            "SELECT id, hostId, platformId FROM context"
            " WHERE branchId = %s AND sha1Id = %s",
            (branch_id, sha1_id))
        for context_id, host_id, platform_id in contexts:
            uploads = execQuery(
                "SELECT count(DISTINCT uploadId) FROM result"
                " WHERE contextId = %s",
                (context_id,))[0][0]

            # Create 'testsystem' element:
            testsystem_elem = doc.createElement("testsystem")
            testsystem_elem.setAttribute("host", idToText("host", host_id))
            testsystem_elem.setAttribute(
                "platform", idToText("platform", platform_id))
            testsystem_elem.setAttribute("uploads", str(uploads))
            branch_elem.appendChild(testsystem_elem)
    # --- END create XML structure ---


    # Dump XML structure to local file:
    file_name = "bmtestable.xml"
    tmp_abs_fpath = "/tmp/" + file_name
    f = open(tmp_abs_fpath, 'w')
    f.write(doc.toprettyxml(indent='    '))
    f.close()

    repo_dir = "/home/qt/bmtestable"

    if remote: # script runs on a different host than gitestr.nokia.troll.no

        gitestr_user_at_host = "qt@gitestr.nokia.troll.no"

        # Install file:
        # (note: copying the file to the repository is done in two steps
        # in order to minimize the likelihood of a client reading an
        # incomplete file)
        repo_dir = "/home/qt/bmtestable"
        if not runCommand(
            ["scp", tmp_abs_fpath, gitestr_user_at_host + ":" + tmp_abs_fpath]):
            sys.stderr.write("exiting ...\n")
            sys.exit(1)
        runCommand(["rm", "-f", tmp_abs_fpath]) # don't need this anymore
        if not runCommand(
            ["ssh", gitestr_user_at_host,
             "mv " + tmp_abs_fpath + " " + repo_dir]):
            sys.stderr.write("exiting ...\n")
            sys.exit(1)

        # Commit (for source control):
        if not runCommand(
            ["ssh", gitestr_user_at_host, "cd " + repo_dir,
             "; git commit -a -m update"]): # Note semicolon!
            sys.stderr.write(
                file_name + " was expected to change, but apparently did not\n")
            sys.stderr.write("exiting ...\n")
            sys.exit(1)

    else: # script runs locally on gitestr.nokia.troll.no

        # Install file:
        if not runCommand(["mv", tmp_abs_fpath, "."], repo_dir):
            sys.stderr.write("exiting ...\n")
            sys.exit(1)

        # Commit (for source control):
        if not runCommand(["git", "commit", "-a", "-m", "update"], repo_dir):
            sys.stderr.write(
                file_name + " was expected to change, but apparently did not\n")
            sys.stderr.write("exiting ...\n")
            sys.exit(1)



# --- END Global functions ----------------------------------------------


# --- BEGIN Main program ----------------------------------------------

options, http_get = getOptions()

if "help" in options:
    printVerboseUsage()
    sys.exit(1)

if (not ("db" in options and "host" in options and "platform" in options and
    "branch" in options and "sha1" in options and "file" in options)):
    printUsage()
    sys.exit(1)

if not isValidSHA1(options["sha1"]):
    sys.stderr.write("error: invalid SHA-1: " + options["sha1"] + "\n")
    sys.exit(1)

setDatabase(
    options["dbhost"] if "dbhost" in options else None,
    options["dbport"] if "dbport" in options else None,
    options["db"])


sys.stdout.write("UPLOADING RESULTS, OPTIONS: " + str(options) + "\n")
sys.stdout.flush()


# Reject uploading if this context is already complete (since it could
# modify the input for (and thus invalidate) any rankings computed for
# this snapshot):
if contextComplete(options):
    sys.stderr.write(
        "this snapshot is already complete for this time series -> uploading "
        "rejected!\n")
    sys.exit(1)


# Reject uploading if rankings exist for this context, since that would
# require a recomputation of those rankings.
#
# (Note that we only need to check the current SHA-1 as long as we assume that
# results are uploaded in an order that is consistent with the order of the
# corresponding SHA-1s in the branch in question: Rankings for earlier SHA-1s
# will not be affected by results for this SHA-1, and rankings for later
# SHA-1s cannot exist a this point (by assumption).)
if rankingsExist(options):
    sys.stderr.write(
        "error: rankings have already been computed for this context\n")
    sys.exit(1)


# If this is the first set of results for the current context, we compute
# rankings for the same context, only one snapshot back:
if not contextExists(options):
    sys.stdout.write(
        "new snapshot in this time series -> attempt to compute rankings "
        "for the previous snapshot ...\n")
    sys.stdout.flush()
    execComputeRankings(options, True)
else:
    sys.stdout.write(
        "results for this snapshot have already been uploaded -> don't "
        "attempt to compute rankings at this point\n")
    sys.stdout.flush()


# Parse results and store them in the database:
sys.stdout.write("parsing results and uploading to database ... ")
sys.stdout.flush()
try:
    results = extractResults(options["file"])
except BaseException as e:
    sys.stderr.write(
        "error: failed to parse XML file: " + str(e.args[0]) + "\n")
    sys.exit(1)
uploadToDatabase(
    options["host"], options["platform"], options["branch"], options["sha1"],
    results)
sys.stdout.write("done\n")
sys.stdout.flush()


# If no more results are expected in this context, we can compute rankings
# already at this point. (Note: In the case that one or more uploads failed
# for this context, it will be regarded as incomplete forever. In that case,
# computation of rankings for this context will be instead be triggered right
# before uploading the first set of results for the next context (see code
# above).)
if contextComplete(options):
    sys.stdout.write(
        "this snapshot is complete for this time series -> attempt to "
        "compute rankings for this snapshot ...\n")
    sys.stdout.flush()
    execComputeRankings(options, False)
else:
    sys.stdout.write(
        "this snapshot is incomplete for this time series -> "
        "don't attempt to compute rankings at this point\n")


sys.stdout.write("UPLOADING RESULTS DONE\n")


# Update the 'testable snapshots' file for external testers to ensure that they
# test a subset of the snapshots that we test locally:
if options["db"] == "bm": # this is only supported for the 'bm' database
    sys.stdout.write(
        "updating 'testable snapshots' file for external testers ...")
    sys.stdout.flush()
    updateTestableSnapshotsFile(
        ("remote" in options) and (options["remote"] == "1"))
    sys.stdout.write("done\n")


sys.exit(0)

# --- END Main program ----------------------------------------------