From 4136e775df6670906b567456bfa2450556c63128 Mon Sep 17 00:00:00 2001 From: Jo Asplin Date: Tue, 20 Sep 2011 11:51:24 +0200 Subject: Updated README + misc fixes/cleanup --- README | 173 +++++++++++++++++++++++++++++++++++++- database/scripts/README | 10 +-- database/scripts/plpgsqlfuncs.sql | 30 ------- database/scripts/privileges.sql | 7 -- database/scripts/tabledefs.sql | 51 +---------- scripts/getstats.py | 1 - 6 files changed, 173 insertions(+), 99 deletions(-) diff --git a/README b/README index 3c5810a..2264587 100644 --- a/README +++ b/README @@ -1,3 +1,172 @@ -Fill in some contents here at some point. -For now, please refer to http://wikis.in.nokia.com/QtSoftware/BM2 +========================================================================= +OVERVIEW +This repository contains BM2 - a system for storing and analyzing +performance testing results. + +The idea is to run performance tests on test machines and upload the +results to a database on a server machine. The database can then be +inspected either from a web application or using SQL queries directly. + + +========================================================================= +INSTALLATION + +STEP 1: Install the BM2 directory (containing this README!) under an + arbitrary directory . + + +STEP 2: Install Python 2.6 (with development headers) on both the + server machine and the machine from which results will be uploaded + (NOTE: the latter may or may not be identical to the test + machine). + + http://www.python.org/download/releases/2.6.7/ + + NOTE: Version 2.7 or later may not work due to potential source + incompatibilies! + + +STEP 3: Install a recent version of PostgreSQL (with development + headers) (8.4 or later should work fine). + + http://www.postgresql.org/ + + NOTE: Ensure to install both the server (postgres) and interactive + terminal (psql). + + +STEP 4: Start the database server and (if desirable) arrange for it be + started automatically at boot time (on a Linux system this would + typically mean to add some stuff under /etc/init.d/). + + +STEP 5: Initialize the database: + + $ /database/scripts/reset.py + + +STEP 6: Install a recent version the Apache web server (2.2 or later + should do fine). + + http://httpd.apache.org/ + + +STEP 7: Detect and install missing Python modules: + + $ cd /scripts + $ ./getstats.py + + A missing module will reported as: + + ... + ImportError: No module named + + You are likely to be missing the following modules: + + * statlib (statistics) + http://code.google.com/p/python-statlib + + * psycopg2 (PostgreSQL access) + http://initd.org/psycopg + + It should be straightforward to install both of them by following + the instructions on the respective web pages. + + NOTE: Installing psycopg2 fails if you don't have the appropriate + PostgreSQL client library (libpq) installed. + + +STEP 8: Install a script named 'getstatswrapper' for executing + getstats.py with the appropriate environment. The script + typically looks like this on Linux ( is + /home/qt/repos/bm2 in this example): + + #!/bin/sh + LD_LIBRARY_PATH=/usr/local/pgsql/lib \ + /home/qt/repos/bm2/scripts/getstats.py + + +STEP 9: Configure the Apache web server to 1) allow access to the + /web/ directory, and 2) to allow launching + 'getstatswrapper' as a CGI script (see separate documentation + for the web server - typically this requires changes to + httpd.conf etc.). + + +STEP 10: Start the web server and (if desirable) arrange for it be + started automatically at boot time (on a Linux system this would + typically mean to add some stuff under /etc/init.d/). + + +========================================================================= +USAGE + +--- Producing and uploading results ------------------------------- + +STEP 1: Run tests and record performance measurements in an XML file: + + + ... + + ... + + ... + + ... + + + Notes: + - This is the same format as is generated by a QTestLib program started with the -xml + command-line arguument. + - There is a hierarchichal relationship between test cases, test functions, and data tags: + A test case consists of one or more test functions, which in turn consists of one or more + data tags. Each such combination is considered a "benchmark" and is associated with a + single measurement ("benchmark result"). + - The 'tag' attribute may be empty. + - The 'iterations' attribute is optional with a default value of 1. + - The actual value is value / iterations (in the example above it would be 98 / 4 = 24.5). + + +STEP 2: Upload results to the server using the upload script: + + Typically something like this: + + $ ./uploadresults.py --dbhost ... --dbport ... --db bm --host oslo-bm-mac-mini-64-1 \ + --platform macx-g++ --branch qt/qt-script-v8#master-v8 \ + --sha1 ae15094c6818904b47a53590644992f24e7faa97 --file results.xml + + Run './uploadresults.py --help' for a description of the command-line options! + + Notes: + - Results for different SHA-1s (revisions) must be uploaded in the order in + which they appear the revision history. + - Results may be uploaded multiple times for the same SHA-1 (this can be useful + to remove outliers since BM2 uses the median value in such a case). + + + +--- Viewing results ------------------------------------ + +--- Option 1: Using web application -------------------------- + +STEP 1: Open the URL + + Open the top-level index.html page, located under /web/, for example: + + http://bmc.test.qt.nokia.com:8080/bm2 + + where bmc.test.qt.nokia.com is the host address, 8080 the Apache port number, + and bm2 a symbolic link to /web/. + + Obviously, this may be configured differently on different installations. + + +STEP 2: Interact with the web app to show various reports. + + + + +--- Option 2: Query the database directly -------------------------- + + diff --git a/database/scripts/README b/database/scripts/README index 7b193cc..e287f56 100644 --- a/database/scripts/README +++ b/database/scripts/README @@ -1,4 +1,4 @@ -STEP 1: Set up the pg_hba.conf file like this: +STEP 1: Set up the pg_hba.conf file something like this: --- BEGIN -------------------------------------------------------------- [...] @@ -26,11 +26,3 @@ host bm bmuser 172.24.90.79/32 trust # barbarella (jaspli STEP 2: Run the reset.py script to initialize an empty database. - - -STEP 3: Run the reset-ranking.py script to initialize an empty 'ranking' - table. - - NOTE: For now this is kept as a separate script in order not - to affect other tables. When the ranking feature has matured, - everything should be merged into the reset.py script. diff --git a/database/scripts/plpgsqlfuncs.sql b/database/scripts/plpgsqlfuncs.sql index 052a54e..b0c1267 100644 --- a/database/scripts/plpgsqlfuncs.sql +++ b/database/scripts/plpgsqlfuncs.sql @@ -1,33 +1,3 @@ --- Inserts or updates a row in the 'ranking' table. -CREATE OR REPLACE FUNCTION merge_ranking( - context1Id_ BIGINT, context2Id_ BIGINT, benchmarkId_ BIGINT, - metricId_ BIGINT, lastChangeTimestamp_ INTEGER, statId_ BIGINT, - value_ REAL, pos_ BIGINT) RETURNS VOID AS -$$ -BEGIN - BEGIN - INSERT INTO ranking( - context1Id, context2Id, benchmarkId, metricId, lastChangeTimestamp, - statId, value, pos) - VALUES ( - context1Id_, context2Id_, benchmarkId_, metricId_, - lastChangeTimestamp_, statId_, value_, pos_); - RETURN; - EXCEPTION WHEN unique_violation THEN - UPDATE ranking - SET context1Id = context1Id_, - lastChangeTimestamp = lastChangeTimestamp_, value = value_, - pos = pos_ - WHERE context2Id = context2Id_ - AND benchmarkId = benchmarkId_ - AND metricId = metricId_ - AND statId = statId_; - END; -END; -$$ -LANGUAGE plpgsql; - - -- Inserts or updates a row in the 'timeSeriesAnnotation' table. CREATE OR REPLACE FUNCTION merge_timeSeriesNote( hostId_ BIGINT, platformId_ BIGINT, branchId_ BIGINT, diff --git a/database/scripts/privileges.sql b/database/scripts/privileges.sql index 20fa970..24c848b 100644 --- a/database/scripts/privileges.sql +++ b/database/scripts/privileges.sql @@ -50,13 +50,6 @@ GRANT SELECT ON context TO bmuser; GRANT INSERT ON context TO bmuser; GRANT UPDATE ON context_id_seq TO bmuser; -GRANT SELECT ON rankingStat TO bmuser; - -GRANT SELECT ON ranking TO bmuser; -GRANT INSERT ON ranking TO bmuser; -GRANT UPDATE ON ranking TO bmuser; -GRANT UPDATE ON ranking_id_seq TO bmuser; - GRANT SELECT ON timeSeriesAnnotation TO bmuser; GRANT INSERT ON timeSeriesAnnotation TO bmuser; GRANT UPDATE ON timeSeriesAnnotation TO bmuser; diff --git a/database/scripts/tabledefs.sql b/database/scripts/tabledefs.sql index 07f2f84..1b4f114 100644 --- a/database/scripts/tabledefs.sql +++ b/database/scripts/tabledefs.sql @@ -92,55 +92,6 @@ CREATE INDEX result_metric_idx ON result (metricId); CREATE INDEX result_valid_idx ON result (valid); --- Ranking statistic types: -CREATE TABLE rankingStat ( - id BIGSERIAL PRIMARY KEY, value TEXT UNIQUE NOT NULL) WITH (OIDS=FALSE); -ALTER TABLE rankingStat OWNER TO postgres; -INSERT INTO rankingStat (value) VALUES - ('QS'), -- Quality score - ('LCSSR'), -- Last chance stability score (last change regressing) - ('LCSSI'), -- Last change stability score (last change improving) - ('LCSS1R'), -- Last change stability score not considering history after - -- last change (last change regressing) - ('LCSS1I'); -- Last change stability score not considering history after - -- last change (last change improving) - - --- Rankings: -CREATE TABLE ranking -( - id BIGSERIAL PRIMARY KEY, - - -- Contexts refering to start and end of ranking computation interval: - context1Id BIGINT NOT NULL REFERENCES context ON DELETE CASCADE, - context2Id BIGINT NOT NULL REFERENCES context ON DELETE CASCADE, - - benchmarkId BIGINT NOT NULL REFERENCES benchmark ON DELETE CASCADE, - metricId BIGINT NOT NULL REFERENCES metric ON DELETE CASCADE, - - lastChangeTimestamp INTEGER NOT NULL, - - -- Ranking statistic type: - statId BIGINT NOT NULL REFERENCES rankingStat ON DELETE CASCADE, - - -- Ranking statistic value: - value REAL NOT NULL, - - -- Ranking position: - pos INTEGER NOT NULL, - - UNIQUE (context2Id, benchmarkId, metricId, statId) -) WITH (OIDS=FALSE); -ALTER TABLE ranking OWNER TO postgres; -CREATE INDEX ranking_context1_idx ON ranking (context1Id); -CREATE INDEX ranking_context2_idx ON ranking (context2Id); -CREATE INDEX ranking_benchmark_idx ON ranking (benchmarkId); -CREATE INDEX ranking_metric_idx ON ranking (metricId); -CREATE INDEX ranking_lctimestamp_idx ON ranking (lastChangeTimestamp); -CREATE INDEX ranking_stat_idx ON ranking (statId); -CREATE INDEX ranking_pos_idx ON ranking (pos); - - -- Time series annotations: CREATE TABLE timeSeriesAnnotation ( @@ -158,7 +109,7 @@ CREATE TABLE timeSeriesAnnotation UNIQUE (hostId, platformId, branchId, benchmarkId, metricId) ) WITH (OIDS=FALSE); -ALTER TABLE ranking OWNER TO postgres; +ALTER TABLE timeSeriesAnnotation OWNER TO postgres; CREATE INDEX tsanno_host_idx ON timeSeriesAnnotation (hostId); CREATE INDEX tsanno_platform_idx ON timeSeriesAnnotation (platformId); CREATE INDEX tsanno_branch_idx ON timeSeriesAnnotation (branchId); diff --git a/scripts/getstats.py b/scripts/getstats.py index 735b28a..4432501 100755 --- a/scripts/getstats.py +++ b/scripts/getstats.py @@ -59,7 +59,6 @@ def createCommand(options, http_get): "--durtolmax T --benchmark BM --metric M | \\\n" " --cmd snapshots --host H --platform P " "--branch B --sha11 S --sha12 S | \\\n" - "[--maxsize M] | \\\n" " --cmd settimeseriesnote --host H --platform P " "--branch B --benchmark B --metric M --note N | \\\n" " --cmd topchanges --regressions R --last L --timescope T " -- cgit v1.2.3