MigrateToNoteDb: add option to force state change with skipped project

In one of our sites we have a giant repository. When trying to migrate
the complete site using offline migration including this giant repo
the migration crawled at few change/sec since it seems to allocate so
much memory that the migration of the other 25k repositories can’t
really make progress when running the migration with 1 thread/core.
This also caused the JVM GC ratio to increase heavily (>60%). Reducing
the number of threads to 16 reduced the gc ratio to 15-20% but still
migration speed reached only 20 changes/sec.

Hence we migrate this giant repo on a staging copy of the site which
takes around 11 hours when only migrating the huge repository
using 16 threads. Then the meta refs from the giant repo are transferred
from the staging site via git fetch. This is possible since
the repository is read-only so we can be sure there are no new changes
on the production server since we migrated it on the staging server.
With this patch series we can migrate the other 25k repos / 4.5m changes
in a bit more than 1 hour (1200 changes/sec) if we skip the giant
repository in this migration run.

This change adds an option to the offline migration which allows to
finish the migration of the other repositories despite we set the option
to skip the giant repository which wasn't supported before. We must skip
it even when all the migration work was already done on the staging site
and the result transferred to the site where the migration of all the
other repositories is done since otherwise we are back to a very slow
20 changes/second.

An alternative approach could be to migrate this huge repository slowly
using online migration but this would have the disadvantage that this
would take much longer and affect the performance of the productive
server until this migration finished.

Change-Id: Ib78d257ce19bf8370ae0c259d887c600e7195dab

2 files changed, 44 insertions, 4 deletions

diff --git a/java/com/google/gerrit/pgm/MigrateToNoteDb.java b/java/com/google/gerrit/pgm/MigrateToNoteDb.java
index 9bd00fcb92..247aaf9f64 100644
--- a/java/com/google/gerrit/pgm/MigrateToNoteDb.java
+++ b/java/com/google/gerrit/pgm/MigrateToNoteDb.java
@@ -84,6 +84,11 @@ public class MigrateToNoteDb extends SiteProgram {
               + " were previously migrated")
   private boolean force;
 
+  @Option(
+      name = "--force-state-change-with-skip",
+      usage = "Force state change of the migration if projects are skipped")
+  private boolean forceStateChangeWithSkip;
+
   @Option(name = "--trial", usage = TRIAL_USAGE)
   private boolean trial;
 
@@ -145,10 +150,13 @@ public class MigrateToNoteDb extends SiteProgram {
               .setChanges(changes.stream().map(Change.Id::new).collect(toList()))
               .setTrialMode(trial)
               .setForceRebuild(force)
+              .setForceStateChangeWithSkip(forceStateChangeWithSkip)
               .setSequenceGap(sequenceGap)
               .setVerbose(verbose)
               .build()) {
-        if (!projects.isEmpty() || !changes.isEmpty() || !skipProjects.isEmpty()) {
+        if (!projects.isEmpty()
+            || !changes.isEmpty()
+            || (!forceStateChangeWithSkip && !skipProjects.isEmpty())) {
           migrator.rebuild();
         } else {
           migrator.migrate();
diff --git a/java/com/google/gerrit/server/notedb/rebuild/NoteDbMigrator.java b/java/com/google/gerrit/server/notedb/rebuild/NoteDbMigrator.java
index 30b2e12b18..36b07e51ad 100644
--- a/java/com/google/gerrit/server/notedb/rebuild/NoteDbMigrator.java
+++ b/java/com/google/gerrit/server/notedb/rebuild/NoteDbMigrator.java
@@ -183,6 +183,7 @@ public class NoteDbMigrator implements AutoCloseable {
     private NotesMigrationState stopAtState;
     private boolean trial;
     private boolean forceRebuild;
+    private boolean forceStateChangeWithSkip;
     private int sequenceGap = -1;
     private boolean autoMigrate;
     private boolean verbose;
@@ -350,6 +351,21 @@ public class NoteDbMigrator implements AutoCloseable {
     }
 
     /**
+     * Force state change to next migration state if some projects were skipped.
+     *
+     * <p>This makes sense when the skipped projects are migrated in a copy of the site and migrated
+     * data will be transported using git fetch.
+     *
+     * @param forceStateChangeWithSkip whether state change to next migration state should be
+     *     enforced if some projects were skipped.
+     * @return this.
+     */
+    public Builder setForceStateChangeWithSkip(boolean forceStateChangeWithSkip) {
+      this.forceStateChangeWithSkip = forceStateChangeWithSkip;
+      return this;
+    }
+
+    /**
      * Gap between ReviewDb change sequence numbers and NoteDb.
      *
      * <p>If NoteDb sequences are enabled in a running server, there is a race between the migration
@@ -427,6 +443,7 @@ public class NoteDbMigrator implements AutoCloseable {
           projectCache,
           trial,
           forceRebuild,
+          forceStateChangeWithSkip,
           sequenceGap >= 0 ? sequenceGap : Sequences.getChangeSequenceGap(cfg),
           autoMigrate,
           verbose);
@@ -489,6 +506,7 @@ public class NoteDbMigrator implements AutoCloseable {
   private final NotesMigrationState stopAtState;
   private final boolean trial;
   private final boolean forceRebuild;
+  private final boolean forceStateChangeWithSkip;
   private final int sequenceGap;
   private final boolean autoMigrate;
   private final boolean verbose;
@@ -520,6 +538,7 @@ public class NoteDbMigrator implements AutoCloseable {
       ProjectCache projectCache,
       boolean trial,
       boolean forceRebuild,
+      boolean forceStateChangeWithSkip,
       int sequenceGap,
       boolean autoMigrate,
       boolean verbose)
@@ -555,6 +574,7 @@ public class NoteDbMigrator implements AutoCloseable {
     this.stopAtState = stopAtState;
     this.trial = trial;
     this.forceRebuild = forceRebuild;
+    this.forceStateChangeWithSkip = forceStateChangeWithSkip;
     this.sequenceGap = sequenceGap;
     this.autoMigrate = autoMigrate;
     this.verbose = verbose;
@@ -571,7 +591,9 @@ public class NoteDbMigrator implements AutoCloseable {
   }
 
   public void migrate() throws OrmException, IOException {
-    if (!changes.isEmpty() || !projects.isEmpty() || !skipProjects.isEmpty()) {
+    if (!changes.isEmpty()
+        || !projects.isEmpty()
+        || (!forceStateChangeWithSkip && !skipProjects.isEmpty())) {
       throw new MigrationException(
           "Cannot set changes or projects or skipProjects during full migration; call rebuild()"
               + " instead");
@@ -685,7 +707,9 @@ public class NoteDbMigrator implements AutoCloseable {
   private NotesMigrationState setNoteDbPrimary(NotesMigrationState prev)
       throws MigrationException, OrmException, IOException {
     checkState(
-        projects.isEmpty() && changes.isEmpty() && skipProjects.isEmpty(),
+        projects.isEmpty()
+            && changes.isEmpty()
+            && (forceStateChangeWithSkip || skipProjects.isEmpty()),
         "Should not have attempted setNoteDbPrimary with a subset of changes");
     checkState(
         prev == READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY
@@ -701,7 +725,15 @@ public class NoteDbMigrator implements AutoCloseable {
     logger.atInfo().log("Setting primary storage to NoteDb");
     List<Change.Id> allChanges;
     try (ReviewDb db = unwrapDb(schemaFactory.open())) {
-      allChanges = Streams.stream(db.changes().all()).map(Change::getId).collect(toList());
+      if (forceStateChangeWithSkip) {
+        allChanges =
+            Streams.stream(db.changes().all())
+                .filter(c -> !skipProjects.contains(c.getProject()))
+                .map(Change::getId)
+                .collect(toList());
+      } else {
+        allChanges = Streams.stream(db.changes().all()).map(Change::getId).collect(toList());
+      }
     }
 
     try (ContextHelper contextHelper = new ContextHelper()) {