Use dedicated database connection for every schema migration

In some schema migrations pure database schema update machinery was
misused for long running migrations that entirely unrelated to database
schema upgrades. Most notably Schema_144 that migrating external ids to
NoteDb and Schema_146 where all accounts were migrated from ReviewDb to
NoteDb. How the code is currently organized, that approach only works
for small to medium gerrit installation sites with couple of thousands
users.

This change adjusts the schema update connection management and pass
in schema factory instead ReviewDb instance to the schema migration
process. That way it is possible to open two new connections for each
schema migration step:

1. migrate data
2. bump new schema version

As the consequence of this refactoring big sites shouldn't run into
wait timeout exception during schema migration process.

Bug: Issue 12637
Change-Id: I9bbdd9a09898c4ef23362853c654a39934c3967c

2 files changed, 53 insertions, 34 deletions

diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaUpdater.java b/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaUpdater.java
index 266fbaa894..d45781e540 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaUpdater.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaUpdater.java
@@ -102,24 +102,28 @@ public class SchemaUpdater {
   }
 
   public void update(UpdateUI ui) throws OrmException {
+    CurrentSchemaVersion version;
+    SchemaVersion u;
     try (ReviewDb db = ReviewDbUtil.unwrapDb(schema.open())) {
-
-      final SchemaVersion u = updater.get();
-      final CurrentSchemaVersion version = getSchemaVersion(db);
+      version = getSchemaVersion(db);
+      u = updater.get();
       if (version == null) {
         try {
           creator.create(db);
         } catch (IOException | ConfigInvalidException e) {
           throw new OrmException("Cannot initialize schema", e);
         }
+      }
+    }
 
-      } else {
-        try {
-          u.check(ui, version, db);
-        } catch (SQLException e) {
-          throw new OrmException("Cannot upgrade schema", e);
-        }
+    if (version != null) {
+      try {
+        u.check(ui, version, schema);
+      } catch (SQLException e) {
+        throw new OrmException("Cannot upgrade schema", e);
+      }
 
+      try (ReviewDb db = ReviewDbUtil.unwrapDb(schema.open())) {
         updateSystemConfig(db);
       }
     }
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaVersion.java b/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaVersion.java
index d1cbad62b2..5681ecee01 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaVersion.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaVersion.java
@@ -19,9 +19,11 @@ import com.google.common.base.Stopwatch;
 import com.google.common.collect.Lists;
 import com.google.gerrit.reviewdb.client.CurrentSchemaVersion;
 import com.google.gerrit.reviewdb.server.ReviewDb;
+import com.google.gerrit.reviewdb.server.ReviewDbUtil;
 import com.google.gwtorm.jdbc.JdbcExecutor;
 import com.google.gwtorm.jdbc.JdbcSchema;
 import com.google.gwtorm.server.OrmException;
+import com.google.gwtorm.server.SchemaFactory;
 import com.google.gwtorm.server.StatementExecutor;
 import com.google.inject.Provider;
 import java.sql.PreparedStatement;
@@ -68,7 +70,7 @@ public abstract class SchemaVersion {
     return prior.get();
   }
 
-  public final void check(UpdateUI ui, CurrentSchemaVersion curr, ReviewDb db)
+  public final void check(UpdateUI ui, CurrentSchemaVersion curr, SchemaFactory<ReviewDb> schema)
       throws OrmException, SQLException {
     if (curr.versionNbr == versionNbr) {
       // Nothing to do, we are at the correct schema.
@@ -80,35 +82,41 @@ public abstract class SchemaVersion {
               + versionNbr
               + ".");
     } else {
-      upgradeFrom(ui, curr, db);
+      upgradeFrom(ui, curr, schema);
     }
   }
 
   /** Runs check on the prior schema version, and then upgrades. */
-  private void upgradeFrom(UpdateUI ui, CurrentSchemaVersion curr, ReviewDb db)
+  private void upgradeFrom(UpdateUI ui, CurrentSchemaVersion curr, SchemaFactory<ReviewDb> schema)
       throws OrmException, SQLException {
     List<SchemaVersion> pending = pending(curr.versionNbr);
-    updateSchema(pending, ui, db);
-    migrateData(pending, ui, curr, db);
 
-    JdbcSchema s = (JdbcSchema) db;
-    final List<String> pruneList = new ArrayList<>();
-    s.pruneSchema(
-        new StatementExecutor() {
-          @Override
-          public void execute(String sql) {
-            pruneList.add(sql);
-          }
-
-          @Override
-          public void close() {
-            // Do nothing.
-          }
-        });
+    try (ReviewDb db = ReviewDbUtil.unwrapDb(schema.open())) {
+      updateSchema(pending, ui, db);
+    }
 
-    try (JdbcExecutor e = new JdbcExecutor(s)) {
-      if (!pruneList.isEmpty()) {
-        ui.pruneSchema(e, pruneList);
+    migrateData(pending, ui, curr, schema);
+
+    try (ReviewDb db = ReviewDbUtil.unwrapDb(schema.open())) {
+      JdbcSchema s = (JdbcSchema) db;
+      List<String> pruneList = new ArrayList<>();
+      s.pruneSchema(
+          new StatementExecutor() {
+            @Override
+            public void execute(String sql) {
+              pruneList.add(sql);
+            }
+
+            @Override
+            public void close() {
+              // Do nothing.
+            }
+          });
+
+      try (JdbcExecutor e = new JdbcExecutor(s)) {
+        if (!pruneList.isEmpty()) {
+          ui.pruneSchema(e, pruneList);
+        }
       }
     }
   }
@@ -145,13 +153,20 @@ public abstract class SchemaVersion {
   protected void preUpdateSchema(ReviewDb db) throws OrmException, SQLException {}
 
   private void migrateData(
-      List<SchemaVersion> pending, UpdateUI ui, CurrentSchemaVersion curr, ReviewDb db)
+      List<SchemaVersion> pending,
+      UpdateUI ui,
+      CurrentSchemaVersion curr,
+      SchemaFactory<ReviewDb> schema)
       throws OrmException, SQLException {
     for (SchemaVersion v : pending) {
       Stopwatch sw = Stopwatch.createStarted();
       ui.message(String.format("Migrating data to schema %d ...", v.getVersionNbr()));
-      v.migrateData(db, ui);
-      v.finish(curr, db);
+      try (ReviewDb db = ReviewDbUtil.unwrapDb(schema.open())) {
+        v.migrateData(db, ui);
+      }
+      try (ReviewDb db = ReviewDbUtil.unwrapDb(schema.open())) {
+        v.finish(curr, db);
+      }
       ui.message(String.format("\t> Done (%.3f s)", sw.elapsed(TimeUnit.MILLISECONDS) / 1000d));
     }
   }