#!/usr/bin/env python import sys from subprocess import Popen, PIPE from xml.dom.minidom import parse, getDOMImplementation from dbaccess import setDatabase, execQuery, commit from misc import ( getOptions, textToId, idToText, findOrInsertId, isValidSHA1, getContext, getAllSnapshots) # --- BEGIN Global functions ---------------------------------------------- def printUsage(): sys.stderr.write( "usage: " + sys.argv[0] + " --help | [--remote R] [--dbhost H --dbport P] --db D --host H " "--platform P --branch B --sha1 S --file F\n") def printVerboseUsage(): printUsage() sys.stderr.write("\noptions:\n") sys.stderr.write( " --help: This help.\n") sys.stderr.write( " --remote: A flag (0 or 1; default = 0) indicating whether this " "script runs on a different server than gitestr.nokia.troll.no.\n") sys.stderr.write( " --dbhost: The database server host (overriding the default).\n") sys.stderr.write( " --dbport: The database server port (overriding the default).\n") sys.stderr.write( " --db: The database. One of 'bm' or 'bm-dev' (the latter " "intended for experimentation).\n") sys.stderr.write( " --host: The physical machine on which the results were " "produced (e.g. barbarella or 172.24.90.79).\n") sys.stderr.write( "--platform: The OS/compiler/architecture combination " "(e.g. linux-g++-32).\n") sys.stderr.write( " --branch: The product branch (e.g. 'qt 4.6', 'qt 4.7', or " "'qt master').\n") sys.stderr.write( " --sha1: The tested revision within the branch. Can be " "extracted using 'git log -1 --pretty=format:%H' (assuming the " "tested revision is the current head revision).\n") sys.stderr.write( " --file: The results file in QTestLib XML output format.\n") # Returns True iff a low value indicates better performance than a high # value for the given metric. def lowerIsBetter(metric): return { "walltimemilliseconds": True, "walltime": True, "cputicks": True, "instructionreads": True, "events": True, "bitspersecond": False, "bytespersecond": False, "framespersecond": False, "fps": False # add more if necessary ... }[metric.lower()] # Returns the canonical (i.e. "unaliased") form of the metric name. def canonicalMetric(metric): if (metric.lower() == "walltime"): return "WalltimeMilliseconds" if (metric.lower() == "framespersecond"): return "fps" return metric # Returns True iff at least one of the given incidents indicates failure for # the given data tag. def matchesFailedIncident(dataTag, incidents): for incident in incidents: if (incident.getAttribute("type") == "fail"): dataTagElems = incident.getElementsByTagName("DataTag") if dataTag == "": # special case assert len(dataTagElems) == 0 return True try: dataTagElem = dataTagElems[0] except: continue if (dataTagElem.childNodes[0].data == dataTag): return True return False # Returns results extracted from a file in QTestLib XML format # (note: multiple top-level TestCase elements are allowed). def extractResults(file): def processBenchmarkResults( results, dom, testCase, testFuncElem, testFunction, incidents): # Loop over benchmark results ... bmResultElems = testFuncElem.getElementsByTagName("BenchmarkResult") for bmResultElem in bmResultElems: # Data tag (note that "" is a valid data tag): dataTag = bmResultElem.getAttribute("tag").strip() # Metric: metric = bmResultElem.getAttribute("metric").strip() try: lowerIsBetter_ = lowerIsBetter(metric) except: sys.stdout.write( "WARNING: skipping result for unsupported metric: >" + metric + "<\n") continue metric = canonicalMetric(metric) # Value: value = float(bmResultElem.getAttribute("value")) # Iterations (optional): iterAttr = bmResultElem.getAttribute("iterations").strip() if (iterAttr != ""): try: iterations = int(iterAttr) assert iterations > 0 value = value / iterations except: raise BaseException( "found 'iterations' attribute that is not a " + "positive integer: " + iterAttr) # Valid: valid = not matchesFailedIncident(dataTag, incidents) # Add item to array ... results.append( {'testCase': testCase, 'testFunction': testFunction, 'dataTag': dataTag, 'metric': metric, 'lowerIsBetter': lowerIsBetter_, 'value': value, 'valid': valid}) def processTestFunctions(results, dom, testCaseElem, testCase): # Loop over test functions ... testFuncElems = testCaseElem.getElementsByTagName("TestFunction") for testFuncElem in testFuncElems: testFunction = testFuncElem.getAttribute("name").strip() assert testFunction != "" incidents = testFuncElem.getElementsByTagName("Incident") processBenchmarkResults( results, dom, testCase, testFuncElem, testFunction, incidents) def processTestCases(results, dom): # Loop over test cases ... testCaseElems = dom.getElementsByTagName("TestCase") for testCaseElem in testCaseElems: testCase = testCaseElem.getAttribute("name").strip() assert testCase != "" processTestFunctions( results, dom, testCaseElem, testCase) # Load DOM structure from file: try: dom = parse(file) except: raise BaseException(sys.exc_info()) # Extract benchmark results from DOM structure: results = [] processTestCases(results, dom) dom.unlink() return results # Uploads a set of results to the database. def uploadToDatabase(host, platform, branch, sha1, results): # Append a row to the 'upload' table (to record this upload event) ... execQuery("INSERT INTO upload DEFAULT VALUES", (), False) # Retrieve the ID of the row we just inserted ... uploadId = execQuery("SELECT currval('upload_id_seq')", ())[0][0] hostId = findOrInsertId("host", host) platformId = findOrInsertId("platform", platform) branchId = findOrInsertId("branch", branch) sha1Id = findOrInsertId("sha1", sha1) contextId = getContext(hostId, platformId, branchId, sha1Id) if contextId == -1: contextId = execQuery( "INSERT INTO context" " (hostId, platformId, branchId, sha1Id)" " VALUES (%s, %s, %s, %s)" " RETURNING id", (hostId, platformId, branchId, sha1Id))[0][0] # Append rows to the 'result' table ... for result in results: benchmark = ( result['testCase'] + ":" + result['testFunction'] + "(" + str(result['dataTag']) + ")") testCaseId = findOrInsertId("testCase", result['testCase']) benchmarkId = findOrInsertId( "benchmark", benchmark, "testCaseId", testCaseId) metricId = findOrInsertId( "metric", result['metric'], "lowerIsBetter", result['lowerIsBetter']) execQuery( "INSERT INTO result" " (contextId, benchmarkId, value, valid, metricId, uploadId)" " VALUES (%s, %s, %s, %s, %s, %s)", (contextId, benchmarkId, result['value'], result['valid'], metricId, uploadId), False) # Write to database: commit() # Returns the context ID if found, otherwise -1: def getContextIdFromNames(options): host_id = textToId("host", options["host"]) platform_id = textToId("platform", options["platform"]) branch_id = textToId("branch", options["branch"]) sha1_id = textToId("sha1", options["sha1"]) return getContext(host_id, platform_id, branch_id, sha1_id) # Returns True iff this context exists: def contextExists(options): return getContextIdFromNames(options) != -1 # Returns True iff no more results are to be expected for this context: def contextComplete(options): max_sample_size = 5 # WARNING: This value must match the corresponding value # in the script that triggers benchmark execution. context_id = getContextIdFromNames(options) sample_size = execQuery( "SELECT count(*) FROM" " (SELECT DISTINCT uploadId from result where contextId = %s) AS foo", (context_id,))[0][0] return sample_size >= max_sample_size # Executes the external updatechanges.py script with appropriate arguments. def execUpdateChanges(options): cmd = [ "updatechanges.py", "--db", options["db"], "--host", options["host"], "--platform", options["platform"], "--branch", options["branch"], "--noprogress", "true"] if "dbhost" in options: cmd += ["--dbhost", options["dbhost"]] if "dbport" in options: cmd += ["--dbport", options["dbport"]] p = Popen(cmd, stdout = PIPE, stderr = PIPE) stdout, stderr = p.communicate() if (p.returncode != 0): sys.stdout.write("failed to execute command '" + str(cmd) + "':\n") sys.stdout.write(" return code: " + str(p.returncode) + "\n") sys.stdout.write(" stdout: >" + stdout.strip() + "<\n") sys.stdout.write(" stderr: >" + stderr.strip() + "<\n") else: sys.stdout.write("updatechanges.py executed successfully:\n") sys.stdout.write(" return code: " + str(p.returncode) + "\n") sys.stdout.write(" stdout: >" + stdout.strip() + "<\n") sys.stdout.write(" stderr: >" + stderr.strip() + "<\n") # Executes an external command. Returns True iff the return code of the # command is zero. def runCommand(cmd, cwd_ = None): p = Popen(cmd, cwd = cwd_, stdout = PIPE, stderr = PIPE) stdout, stderr = p.communicate() if p.returncode != 0: print "\nfailed to run command", cmd print " return code:", p.returncode print " stdout: >%s<" % stdout.strip() print " stderr: >%s<" % stderr.strip() return False return True # Updates a 'testable snapshots' file on gitestr.nokia.troll.no that can be # used by external test clients to determine the most recent snapshot (SHA-1) # of each branch and also how many uploads have been completed by each # host/platform combination for these branch/snapshot combinations. # # This script is assumed to run on a different host than # gitestr.nokia.troll.no iff the 'remote' argument is True. # def updateTestableSnapshotsFile(remote): # --- BEGIN create XML structure --- impl = getDOMImplementation() doc = impl.createDocument(None, "branches", None) branches_elem = doc.documentElement # Loop over branches: branch_ids = execQuery("SELECT DISTINCT branchId FROM context", ()) for branch_id in zip(*branch_ids)[0]: # Get the latest snapshot for this branch: sha1_id = execQuery( "SELECT sha1Id FROM context WHERE branchId = %s" " ORDER BY timestamp DESC LIMIT 1", (branch_id,))[0][0] # Create 'branch' element: branch_elem = doc.createElement("branch") branch_elem.setAttribute("name", idToText("branch", branch_id)) branch_elem.setAttribute("sha1", idToText("sha1", sha1_id)) branches_elem.appendChild(branch_elem) # Loop over contexts matching this branch/snapshot combination: contexts = execQuery( "SELECT id, hostId, platformId FROM context" " WHERE branchId = %s AND sha1Id = %s", (branch_id, sha1_id)) for context_id, host_id, platform_id in contexts: uploads = execQuery( "SELECT count(DISTINCT uploadId) FROM result" " WHERE contextId = %s", (context_id,))[0][0] # Create 'testsystem' element: testsystem_elem = doc.createElement("testsystem") testsystem_elem.setAttribute("host", idToText("host", host_id)) testsystem_elem.setAttribute( "platform", idToText("platform", platform_id)) testsystem_elem.setAttribute("uploads", str(uploads)) branch_elem.appendChild(testsystem_elem) # --- END create XML structure --- # Dump XML structure to local file: file_name = "bmtestable.xml" tmp_abs_fpath = "/tmp/" + file_name f = open(tmp_abs_fpath, 'w') f.write(doc.toprettyxml(indent=' ')) f.close() repo_dir = "/home/qt/bmtestable" if remote: # script runs on a different host than gitestr.nokia.troll.no gitestr_user_at_host = "qt@gitestr.nokia.troll.no" # Install file: # (note: copying the file to the repository is done in two steps # in order to minimize the likelihood of a client reading an # incomplete file) repo_dir = "/home/qt/bmtestable" if not runCommand( ["scp", tmp_abs_fpath, gitestr_user_at_host + ":" + tmp_abs_fpath]): sys.stderr.write("exiting ...\n") sys.exit(1) runCommand(["rm", "-f", tmp_abs_fpath]) # don't need this anymore if not runCommand( ["ssh", gitestr_user_at_host, "mv " + tmp_abs_fpath + " " + repo_dir]): sys.stderr.write("exiting ...\n") sys.exit(1) # Commit (for source control): if not runCommand( ["ssh", gitestr_user_at_host, "cd " + repo_dir, "; git commit -a -m update"]): # Note semicolon! sys.stderr.write( file_name + " was expected to change, but apparently did not\n") sys.stderr.write("exiting ...\n") sys.exit(1) else: # script runs locally on gitestr.nokia.troll.no # Install file: if not runCommand(["mv", tmp_abs_fpath, "."], repo_dir): sys.stderr.write("exiting ...\n") sys.exit(1) # Commit (for source control): if not runCommand(["git", "commit", "-a", "-m", "update"], repo_dir): sys.stderr.write( file_name + " was expected to change, but apparently did not\n") sys.stderr.write("exiting ...\n") sys.exit(1) # --- END Global functions ---------------------------------------------- # --- BEGIN Main program ---------------------------------------------- options, http_get = getOptions() if "help" in options: printVerboseUsage() sys.exit(1) if (not ("db" in options and "host" in options and "platform" in options and "branch" in options and "sha1" in options and "file" in options)): printUsage() sys.exit(1) if not isValidSHA1(options["sha1"]): sys.stderr.write("error: invalid SHA-1: " + options["sha1"] + "\n") sys.exit(1) setDatabase( options["dbhost"] if "dbhost" in options else None, options["dbport"] if "dbport" in options else None, options["db"]) sys.stdout.write("UPLOADING RESULTS, OPTIONS: " + str(options) + "\n") sys.stdout.flush() # Reject uploading if this context is already complete: if contextComplete(options): sys.stderr.write( "this snapshot is already complete -> uploading rejected!\n") sys.exit(1) # If this is the first set of results for the current context, we update # changes for all time series in this host/platform/branch combination: if not contextExists(options): sys.stdout.write("update changes (before registering new snapshot) ...\n") sys.stdout.flush() execUpdateChanges(options) else: sys.stdout.write("skipping update changes (1)\n") sys.stdout.flush() # Parse results and store them in the database: sys.stdout.write("parsing results and uploading to database ... ") sys.stdout.flush() try: results = extractResults(options["file"]) except BaseException as e: sys.stderr.write( "error: failed to parse XML file: " + str(e.args[0]) + "\n") sys.exit(1) uploadToDatabase( options["host"], options["platform"], options["branch"], options["sha1"], results) sys.stdout.write("done\n") sys.stdout.flush() # If no more results are expected in this context, we can update changes # already at this point. (Note: In the case that one or more uploads failed # for this context, it will be regarded as incomplete forever. In that case, # update of changes for this context will be instead be triggered right # before uploading the first set of results for the next context (see code # above).) if contextComplete(options): sys.stdout.write( "update changes (after completion of current snapshot) ...\n") sys.stdout.flush() execUpdateChanges(options) else: sys.stdout.write("skipping update changes (2)\n") sys.stdout.flush() sys.stdout.write("UPLOADING RESULTS DONE\n") # Update the 'testable snapshots' file for external testers to ensure that they # test a subset of the snapshots that we test locally: if options["db"] == "bm": # this is only supported for the 'bm' database sys.stdout.write( "updating 'testable snapshots' file for external testers ...") sys.stdout.flush() updateTestableSnapshotsFile( ("remote" in options) and (options["remote"] == "1")) sys.stdout.write("done\n") sys.exit(0) # --- END Main program ----------------------------------------------