Cleaned up handling of command-line args ++.

This commit adds optional dbhost and dbport args to
the command line in order to use other database servers
than the default.

5 files changed, 240 insertions, 196 deletions

diff --git a/scripts/dbaccess.py b/scripts/dbaccess.py
index d9e0eb2..d18346e 100644
--- a/scripts/dbaccess.py
+++ b/scripts/dbaccess.py
@@ -1,11 +1,25 @@
 import sys
 import psycopg2
 
-def setDatabase(db_):
-    global db
+def setDatabase(dbhost_, dbport_, db_):
+    global dbhost, dbport, db
+    dbhost = dbhost_
+    dbport = dbport_
     db = db_
+    assert "dbhost" in globals()
+    assert "dbport" in globals()
     assert "db" in globals()
 
+def databaseHost():
+    global dbhost
+    assert "dbhost" in globals()
+    return dbhost
+
+def databasePort():
+    global dbport
+    assert "dbport" in globals()
+    return dbport
+
 def database():
     global db
     assert "db" in globals()
@@ -13,15 +27,19 @@ def database():
 
 # Connects to the database and creates a global cursor:
 def connectDatabase():
+    assert "dbhost" in globals()
+    assert "dbport" in globals()
     assert "db" in globals()
 
+    default_dbhost = "bmc.test.qt.nokia.com" # Default unless overridden
+    default_dbport = 5432 # Default unless overridden
+    assert db != None # Needs to be explicitly specified for now
     try:
-        # ### hardcode for now:
         conn = psycopg2.connect(
-            host="bmc.test.qt.nokia.com",
-            port="5432",
+            host=(dbhost if dbhost else default_dbhost),
+            port=(dbport if dbport else default_dbport),
             database=db,
-            user="bmuser")
+            user="bmuser") # Hardcoded for now
     except:
         print "failed to connect to the database:", sys.exc_info()
         sys.exit(1)
diff --git a/scripts/finalizeresults.py b/scripts/finalizeresults.py
index ec7c156..5e729eb 100755
--- a/scripts/finalizeresults.py
+++ b/scripts/finalizeresults.py
@@ -3,31 +3,43 @@
 import sys
 from dbaccess import setDatabase, execQuery, commit
 from misc import (
-    textToId, getAllSnapshots, getLastRankingSnapshot, getContext, isValidSHA1,
-    getBMTimeSeriesStatsList)
+    getOptions, textToId, getAllSnapshots, getLastRankingSnapshot, getContext,
+    isValidSHA1, getBMTimeSeriesStatsList)
 
 
-# --- BEGIN global functions ------------------------------------------------
+# === Global functions ===============================================
 
 def printUsage():
     print (
         "usage:" + sys.argv[0] +
-        " --help | <host> <platform> <branch> <SHA-1> <database>")
+        " --help | [--dbhost H --dbport P] --db D --host H --platform P "
+        "--branch B --sha1 S")
 
 def printVerboseUsage():
     printUsage()
-    print ("<host>:      The physical machine on which the results were " +
-           "produced (e.g. barbarella or 172.24.90.79).")
-    print ("<platform>:  The OS/compiler/architecture combination " +
-           "(e.g. linux-g++-32).")
-    print ("<branch>:    The product branch (e.g. 'qt 4.6', 'qt 4.7', or " +
-           "'qt master').")
-    print ("<SHA-1>:     The tested revision within the branch. Can be " +
-           "extracted using 'git log -1 --pretty=format:%H' (assuming the " +
-           "tested revision is the current head revision).")
-    print (
-        "<database>:  The database. One of 'bm' or 'bm-dev' (the latter " +
+    print "\noptions:"
+    print(
+        "    --help: This help.")
+    print(
+        "  --dbhost: The database server host (overriding the default).")
+    print(
+        "  --dbport: The database server port (overriding the default).")
+    print(
+        "      --db: The database. One of 'bm' or 'bm-dev' (the latter "
         "intended for experimentation).")
+    print(
+        "    --host: The physical machine on which the results were "
+        "produced (e.g. barbarella or 172.24.90.79).")
+    print(
+        "--platform: The OS/compiler/architecture combination "
+        "(e.g. linux-g++-32).")
+    print(
+        "  --branch: The product branch (e.g. 'qt 4.6', 'qt 4.7', or "
+        "'qt master').")
+    print(
+        "    --sha1: The tested revision within the branch. Can be "
+        "extracted using 'git log -1 --pretty=format:%H' (assuming the "
+        "tested revision is the current head revision).")
 
 
 # ### 2 B DOCUMENTED!
@@ -282,45 +294,44 @@ def updateRankings(host_id, platform_id, branch_id, sha12_id, context2_id):
 
     sys.stdout.write("\n")
 
-# --- END global functions ------------------------------------------------
 
+# === Main program ===================================================
 
-# --- BEGIN main program ------------------------------------------------
+options, http_get = getOptions()
 
-if ((len(sys.argv) > 1) and (sys.argv[1] == "--help")):
+if "help" in options:
     printVerboseUsage()
-    sys.exit(1)
+    sys.exit(0)
 
-if (len(sys.argv) != 6):
+if (not ("db" in options and "host" in options and "platform" in options and
+    "branch" in options and "sha1" in options)):
     printUsage()
-    sys.exit(1)
+    sys.exit(0)
 
-host      = sys.argv[1]
-platform  = sys.argv[2]
-branch    = sys.argv[3]
-sha12     = sys.argv[4]
-if (not isValidSHA1(sha12)):
-    print "invalid SHA-1:", sha12
+if not isValidSHA1(options["sha1"]):
+    print "invalid SHA-1:", options["sha1"]
     sys.exit(1)
-db = sys.argv[5]
 
-setDatabase(db)
+setDatabase(
+    options["dbhost"] if "dbhost" in options else None,
+    options["dbport"] if "dbport" in options else None,
+    options["db"])
 
-host_id = textToId("host", host)
+host_id = textToId("host", options["host"])
 if host_id == -1:
-    print "no such host:", host
+    print "no such host:", options["host"]
     sys.exit(1)
-platform_id = textToId("platform", platform)
+platform_id = textToId("platform", options["platform"])
 if platform_id == -1:
-    print "no such platform:", platform
+    print "no such platform:", options["platform"]
     sys.exit(1)
-branch_id = textToId("branch", branch)
+branch_id = textToId("branch", options["branch"])
 if branch_id == -1:
-    print "no such branch:", branch
+    print "no such branch:", options["branch"]
     sys.exit(1)
-sha12_id = textToId("sha1", sha12)
+sha12_id = textToId("sha1", options["sha1"])
 if sha12_id == -1:
-    print "no such SHA-1:", sha12
+    print "no such SHA-1:", options["sha1"]
     sys.exit(1)
 
 context2_id = getContext(host_id, platform_id, branch_id, sha12_id)
@@ -331,10 +342,7 @@ if context2_id == -1:
 
 updateRankings(host_id, platform_id, branch_id, sha12_id, context2_id)
 
-
 # Write to database:
 commit()
 
 print "finalization done"
-
-# --- END main program ------------------------------------------------
diff --git a/scripts/getstats.py b/scripts/getstats.py
index 5527c46..feb665a 100755
--- a/scripts/getstats.py
+++ b/scripts/getstats.py
@@ -13,84 +13,42 @@ from getrankings import GetRankingsAsJSON
 from settimeseriesnote import SetTimeSeriesNote
 
 from dbaccess import setDatabase
-from misc import printErrorAsJSON
+from misc import getOptions, printErrorAsJSON
 import sys
-import os
-import re
-import urllib
 
 # === Global functions ===============================================
 
-# Returns a 2-tuple consisting of:
-# 1: an option dictionary, and
-# 2: a flag that is true iff the QUERY_STRING environment variable is
-#    present (i.e. that the script is invoked as a CGI-script for a
-#    HTTP GET request).
-#
-# The option dictionary is extracted from either the QUERY_STRING environment
-# variable (first priority) or command-line arguments (second priority).
-# In the latter case, the options must be of the form
-# ... --<opt1> <val1> ... --<optN> <valN> ...
-def getOptions():
-
-    def getOptDictFromQueryString(qs):
-        options = {}
-        for sq in qs.split("&"):
-            keyval = sq.split("=")
-            options[keyval[0]] = urllib.unquote(keyval[1])
-        return options
-
-    def getOptDictFromCommandLine():
-        options = {}
-        p = re.compile("^--(.+)$")
-        key = None
-        for arg in sys.argv[1:]:
-            if key != None:
-                options[key] = arg
-            m = p.match(arg)
-            if m:
-                key = m.group(1)
-            else:
-                key = None
-        return options
-
-    qs = "QUERY_STRING"
-    if qs in os.environ:
-        return (getOptDictFromQueryString(os.environ[qs]), True)
-    else:
-        return (getOptDictFromCommandLine(), False)
-
 # Returns a command instance.
 def createCommand(options, http_get):
 
     def printUsageError():
         error = (
-            "usage: " + sys.argv[0] + " \\\n" +
-            "  --db D --cmd contexts [--rankedonly R] | \\\n" +
-            "  --db D --cmd testcases1 --host H --platform P --branch B " +
-            "--sha1 S | \\\n" +
-            "  --db D --cmd testcases2 --host1 H --platform1 P --branch1 B " +
-            "--sha11 S --host2 H --platform2 P --branch2 B --sha12 S | \\\n" +
-            "  --db D --cmd stats1 --host H --platform P --branch B " +
-            "--sha1 S [--testcasefilter 'TC1 TC2 ...'] | \\\n" +
-            "  --db D --cmd stats2 --host1 H --platform1 P --branch1 B " +
-            "--sha11 S --host2 H --platform2 P --branch2 B --sha12 S " +
-            "[--testcasefilter 'TC1 TC2 ...'] | \\\n" +
-            "  --db D --cmd result_details2 --host1 H --platform1 P " +
-            "--branch1 B --sha11 S --host2 H --platform2 P --branch2 B " +
-            "--sha12 S --benchmark BM --metric M | \\\n" +
-            "  --db D --cmd timeseriesstats --host H --platform P --branch B " +
-            "--sha11 S --sha12 S [--difftol T] [--durtolmin T] " +
-            "[--durtolmax T] [--testcasefilter 'TC1 TC2 ...'] | \\\n" +
-            "  --db D --cmd timeseriesdetails --host H --platform P " +
-            "--branch B --sha11 S --sha12 S --difftol T --durtolmin T " +
-            "--durtolmax T --benchmark BM --metric M | \\\n" +
-            "  --db D --cmd snapshots --host H --platform P " +
-            "--branch B --sha11 S --sha12 S | \\\n" +
-            "  --db D --cmd rankings --host H --platform P " +
-            "--branch B --sha1 S [--testcasefilter 'TC1 TC2 ...'] " +
-            "[--maxsize M] | \\\n" +
-            "  --db D --cmd settimeseriesnote --host H --platform P " +
+            "usage: " + sys.argv[0] + " [--dbhost H --dbport P] --db D + \\\n"
+            "  --cmd contexts [--rankedonly R] | \\\n"
+            "  --cmd testcases1 --host H --platform P --branch B "
+            "--sha1 S | \\\n"
+            "  --cmd testcases2 --host1 H --platform1 P --branch1 B "
+            "--sha11 S --host2 H --platform2 P --branch2 B --sha12 S | \\\n"
+            "  --cmd stats1 --host H --platform P --branch B "
+            "--sha1 S [--testcasefilter 'TC1 TC2 ...'] | \\\n"
+            "  --cmd stats2 --host1 H --platform1 P --branch1 B "
+            "--sha11 S --host2 H --platform2 P --branch2 B --sha12 S "
+            "[--testcasefilter 'TC1 TC2 ...'] | \\\n"
+            "  --cmd result_details2 --host1 H --platform1 P "
+            "--branch1 B --sha11 S --host2 H --platform2 P --branch2 B "
+            "--sha12 S --benchmark BM --metric M | \\\n"
+            "  --cmd timeseriesstats --host H --platform P --branch B "
+            "--sha11 S --sha12 S [--difftol T] [--durtolmin T] "
+            "[--durtolmax T] [--testcasefilter 'TC1 TC2 ...'] | \\\n"
+            "  --cmd timeseriesdetails --host H --platform P "
+            "--branch B --sha11 S --sha12 S --difftol T --durtolmin T "
+            "--durtolmax T --benchmark BM --metric M | \\\n"
+            "  --cmd snapshots --host H --platform P "
+            "--branch B --sha11 S --sha12 S | \\\n"
+            "  --cmd rankings --host H --platform P "
+            "--branch B --sha1 S [--testcasefilter 'TC1 TC2 ...'] "
+            "[--maxsize M] | \\\n"
+            "  --cmd settimeseriesnote --host H --platform P "
             "--branch B --benchmark B --metric M --note N")
 
         if http_get:
@@ -100,8 +58,10 @@ def createCommand(options, http_get):
 
     # Check for mandatory 'db' argument:
     if "db" in options:
-        db = options["db"]
-        setDatabase(db)
+        setDatabase(
+            options["dbhost"] if "dbhost" in options else None,
+            options["dbport"] if "dbport" in options else None,
+            options["db"])
     else:
         printUsageError()
         sys.exit(1)
diff --git a/scripts/misc.py b/scripts/misc.py
index 94e378e..92d4b2f 100644
--- a/scripts/misc.py
+++ b/scripts/misc.py
@@ -1,7 +1,10 @@
 from dbaccess import execQuery
 from statlib import stats
 from string import hexdigits
+import sys
+import os
 import re
+import urllib
 
 # ### 2 B DOCUMENTED!
 def idToText(table, id_):
@@ -661,3 +664,46 @@ def printJSONHeader():
 def printErrorAsJSON(error):
     printJSONHeader()
     print "{\"error\": \"" + error + "\"}\n"
+
+# Returns a 2-tuple consisting of:
+# 1: an option dictionary, and
+# 2: a flag that is true iff the QUERY_STRING environment variable is
+#    present (i.e. that the script is invoked as a CGI-script for a
+#    HTTP GET request).
+#
+# The option dictionary is extracted from either the QUERY_STRING environment
+# variable (first priority) or command-line arguments (second priority).
+# In the latter case, the options must be of the form
+# ... --<opt1> <val1> ... --<optN> <valN> ...
+def getOptions():
+
+    def getOptDictFromQueryString(qs):
+        options = {}
+        for sq in qs.split("&"):
+            keyval = sq.split("=")
+            options[keyval[0]] = urllib.unquote(keyval[1])
+        return options
+
+    def getOptDictFromCommandLine():
+        options = {}
+        p = re.compile("^--(.+)$")
+        key = None
+        for arg in sys.argv[1:]:
+            if key != None:
+                options[key] = arg
+            m = p.match(arg)
+            if m:
+                key = m.group(1)
+                # Support "--help" as the only value-less option:
+                if key == "help":
+                    options[key] = 1
+                    key = None
+            else:
+                key = None
+        return options
+
+    qs = "QUERY_STRING"
+    if qs in os.environ:
+        return (getOptDictFromQueryString(os.environ[qs]), True)
+    else:
+        return (getOptDictFromCommandLine(), False)
diff --git a/scripts/uploadresults.py b/scripts/uploadresults.py
index e26ec1a..9d22315 100755
--- a/scripts/uploadresults.py
+++ b/scripts/uploadresults.py
@@ -3,7 +3,7 @@
 import sys
 from xml.dom.minidom import parse
 from dbaccess import setDatabase, execQuery, commit
-from misc import isValidSHA1, getContext
+from misc import getOptions, isValidSHA1, getContext
 
 
 # === Global functions ===============================================
@@ -11,23 +11,36 @@ from misc import isValidSHA1, getContext
 def printUsage():
     print (
         "usage:" + sys.argv[0] +
-        " --help | <host> <platform> <branch> <SHA-1> <XML file> <database>")
+        " --help | [--dbhost H --dbport P] --db D --host H --platform P "
+        "--branch B --sha1 S --file F")
 
-def printVerboseUsagebbbbq():
+def printVerboseUsage():
     printUsage()
-    print ("<host>:      The physical machine on which the results were " +
-           "produced (e.g. barbarella or 172.24.90.79).")
-    print ("<platform>:  The OS/compiler/architecture combination " +
-           "(e.g. linux-g++-32).")
-    print ("<branch>:    The product branch (e.g. 'qt 4.6', 'qt 4.7', or " +
-           "'qt master').")
-    print ("<SHA-1>:     The tested revision within the branch. Can be " +
-           "extracted using 'git log -1 --pretty=format:%H' (assuming the " +
-           "tested revision is the current head revision).")
-    print ("<XML file>:  The results file (QTestLib output format).")
-    print (
-        "<database>:  The database. One of 'bm' or 'bm-dev' (the latter " +
+    print "\noptions:"
+    print(
+        "    --help: This help.")
+    print(
+        "  --dbhost: The database server host (overriding the default).")
+    print(
+        "  --dbport: The database server port (overriding the default).")
+    print(
+        "      --db: The database. One of 'bm' or 'bm-dev' (the latter "
         "intended for experimentation).")
+    print(
+        "    --host: The physical machine on which the results were "
+        "produced (e.g. barbarella or 172.24.90.79).")
+    print(
+        "--platform: The OS/compiler/architecture combination "
+        "(e.g. linux-g++-32).")
+    print(
+        "  --branch: The product branch (e.g. 'qt 4.6', 'qt 4.7', or "
+        "'qt master').")
+    print(
+        "    --sha1: The tested revision within the branch. Can be "
+        "extracted using 'git log -1 --pretty=format:%H' (assuming the "
+        "tested revision is the current head revision).")
+    print(
+        "    --file: The results file in QTestLib XML output format.")
 
 # Returns True iff a low value indicates better performance than a high
 # value for the given metric.
@@ -140,7 +153,7 @@ def extractResults(file):
 
     # Load DOM structure from file:
     try:
-        dom = parse(xmlFile)
+        dom = parse(file)
     except:
         raise BaseException(sys.exc_info())
 
@@ -179,83 +192,82 @@ def findOrInsertId(table, value, *args):
     return query_result[0][0]
 
 
-# === Main program ===================================================
+# Uploads a set of results to the database.
+def uploadToDatabase(dbhost, dbport, db, host, platform, branch, sha1, results):
 
-# --- Validate arguments -------------------------------------
+    setDatabase(dbhost, dbport, db)
 
-if ((len(sys.argv) > 1) and (sys.argv[1] == "--help")):
-    printVerboseUsage()
-    sys.exit(1)
-
-if (len(sys.argv) != 7):
-    printUsage()
-    sys.exit(1)
+    # Append a row to the 'upload' table (to record this upload event) ...
+    execQuery("INSERT INTO upload DEFAULT VALUES;", False)
 
-host      = sys.argv[1]
-platform  = sys.argv[2]
-branch    = sys.argv[3]
-sha1      = sys.argv[4]
-if (not isValidSHA1(sha1)):
-    print "invalid SHA-1:", sha1
-    sys.exit(1)
-xmlFile = sys.argv[5]
-db      = sys.argv[6]
+    # Retrieve the ID of the row we just inserted ...
+    uploadId = execQuery("SELECT currval('upload_id_seq');")[0][0]
 
-# Perform a basic validation of the other arguments as well ... 2 B DONE!
+    hostId = findOrInsertId("host", host)
+    platformId = findOrInsertId("platform", platform)
+    branchId = findOrInsertId("branch", branch)
+    sha1Id = findOrInsertId("sha1", sha1)
 
-try:
-    results = extractResults(xmlFile)
-except BaseException as e:
-    print "failed to parse XML file:", e.args[0]
-    sys.exit(1)
-#print results
+    contextId = getContext(hostId, platformId, branchId, sha1Id)
+    if contextId == -1:
+        contextId = execQuery(
+            "INSERT INTO context"
+            " (hostId, platformId, branchId, sha1Id)"
+            " VALUES (%d, %d, %d, %d)"
+            " RETURNING id;"
+            % (hostId, platformId, branchId, sha1Id))[0][0]
+
+    # Append rows to the 'result' table ...
+    for result in results:
+        benchmark = (
+            result['testCase'] + ":" + result['testFunction'] + "(" +
+            str(result['dataTag']) + ")")
+        benchmarkId = findOrInsertId("benchmark", benchmark)
+
+        metricId = findOrInsertId(
+            "metric", result['metric'], "lowerIsBetter",
+            result['lowerIsBetter'])
+
+        query = (
+            "INSERT INTO result"
+            " (contextId, benchmarkId, value, valid, metricId, uploadId)"
+            " VALUES (%d, %d, %f, %s, %d, %d);"
+            % (contextId, benchmarkId, result['value'], result['valid'],
+               metricId, uploadId))
 
+        execQuery(query, False)
 
-# --- Upload to database -------------------------------------
+    # Write to database:
+    commit()
 
-setDatabase(db)
 
-# Append a row to the 'upload' table (to record this upload event) ...
-execQuery("INSERT INTO upload DEFAULT VALUES;", False)
+# === Main program ===================================================
 
-# Retrieve the ID of the row we just inserted ...
-uploadId = execQuery("SELECT currval('upload_id_seq');")[0][0]
+options, http_get = getOptions()
 
-hostId = findOrInsertId("host", host)
-platformId = findOrInsertId("platform", platform)
-branchId = findOrInsertId("branch", branch)
-sha1Id = findOrInsertId("sha1", sha1)
+if "help" in options:
+    printVerboseUsage()
+    sys.exit(0)
 
-contextId = getContext(hostId, platformId, branchId, sha1Id)
-if contextId == -1:
-    contextId = execQuery("INSERT INTO context"
-            " (hostId, platformId, branchId, sha1Id)"
-            " VALUES (%d, %d, %d, %d)"
-            " RETURNING id;"
-                % (hostId, platformId, branchId, sha1Id))[0][0]
-
-# Append rows to the 'result' table ...
-for result in results:
-    benchmark = (
-        result['testCase'] + ":" + result['testFunction'] + "(" +
-        str(result['dataTag']) + ")")
-    benchmarkId = findOrInsertId("benchmark", benchmark)
-
-    metricId = findOrInsertId(
-        "metric", result['metric'], "lowerIsBetter",
-        result['lowerIsBetter'])
-
-    query = (
-        "INSERT INTO result"
-        " (contextId, benchmarkId, value, valid, metricId, uploadId)"
-        " VALUES (%d, %d, %f, %s, %d, %d);"
-            % (contextId, benchmarkId, result['value'], result['valid'],
-                metricId, uploadId))
+if (not ("db" in options and "host" in options and "platform" in options and
+    "branch" in options and "sha1" in options and "file" in options)):
+    printUsage()
+    sys.exit(0)
 
-    execQuery(query, False)
+if not isValidSHA1(options["sha1"]):
+    print "invalid SHA-1:", options["sha1"]
+    sys.exit(1)
 
+try:
+    results = extractResults(options["file"])
+except BaseException as e:
+    print "failed to parse XML file:", e.args[0]
+    sys.exit(1)
 
-# Make sure everything is written to the database:
-commit()
+uploadToDatabase(
+    options["dbhost"] if "dbhost" in options else None,
+    options["dbport"] if "dbport" in options else None,
+    options["db"], options["host"], options["platform"], options["branch"],
+    options["sha1"], results)
 
 print "uploading done"