OnlineNoteDbMigration: allow per-project migration

The per-project migration to NoteDb was missing the online
reindexing option which it was making it unusable for
a zero-downtime deployment.

Export the same --project option on the notedb config as
well so that existing sites can start converting to NoteDb
on a per-project basis.

Bug: Issue 14777
Change-Id: I31bcded11ac2c65c492ef7ae1c851954a86318cd

3 files changed, 79 insertions, 10 deletions

diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt
index 223d905393..aff17cac82 100644
--- a/Documentation/config-gerrit.txt
+++ b/Documentation/config-gerrit.txt
@@ -3719,6 +3719,18 @@ each process retrieves at once.
 +
 By default, 1.
 
+[[notedb.onlineMigrationProjects]] notedb.onlineMigrationProjects::
++
+Limit the online migration from reviewDb to noteDb to a list of projects.
++
+This allows to experiment the migration to noteDb on a per-project basis
+for debugging problems and without having to interrupt the service on active
+projects.
++
+When not defined, all projects are migrated to noteDb.
++
+By default, undefined.
+
 [[notedb.onlineMigrationThreads]] notedb.onlineMigrationThreads::
 +
 The number of threads used to run the online migration from reviewDb to noteDb.
diff --git a/java/com/google/gerrit/server/notedb/rebuild/OnlineNoteDbMigrator.java b/java/com/google/gerrit/server/notedb/rebuild/OnlineNoteDbMigrator.java
index 46f5b22e91..fbcc8f5997 100644
--- a/java/com/google/gerrit/server/notedb/rebuild/OnlineNoteDbMigrator.java
+++ b/java/com/google/gerrit/server/notedb/rebuild/OnlineNoteDbMigrator.java
@@ -16,19 +16,24 @@ package com.google.gerrit.server.notedb.rebuild;
 
 import static com.google.gerrit.server.notedb.NotesMigration.SECTION_NOTE_DB;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Stopwatch;
 import com.google.common.flogger.FluentLogger;
 import com.google.gerrit.extensions.events.LifecycleListener;
 import com.google.gerrit.lifecycle.LifecycleModule;
+import com.google.gerrit.reviewdb.client.Project;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.index.OnlineUpgrader;
 import com.google.gerrit.server.index.VersionManager;
+import com.google.gerrit.server.notedb.rebuild.NoteDbMigrator.Builder;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
 import com.google.inject.Singleton;
 import com.google.inject.name.Named;
 import com.google.inject.name.Names;
+import java.util.Arrays;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 import org.eclipse.jgit.lib.Config;
 
 @Singleton
@@ -39,6 +44,8 @@ public class OnlineNoteDbMigrator implements LifecycleListener {
 
   private static final String ONLINE_MIGRATION_THREADS = "onlineMigrationThreads";
 
+  private static final String ONLINE_MIGRATION_PROJECTS = "onlineMigrationProjects";
+
   public static class Module extends LifecycleModule {
     private final boolean trial;
 
@@ -59,6 +66,7 @@ public class OnlineNoteDbMigrator implements LifecycleListener {
   private final boolean upgradeIndex;
   private final boolean trial;
   private final int threads;
+  private final String[] projects;
 
   @Inject
   OnlineNoteDbMigrator(
@@ -73,6 +81,7 @@ public class OnlineNoteDbMigrator implements LifecycleListener {
     this.upgradeIndex = VersionManager.getOnlineUpgrade(cfg);
     this.trial = trial || NoteDbMigrator.getTrialMode(cfg);
     this.threads = cfg.getInt(SECTION_NOTE_DB, ONLINE_MIGRATION_THREADS, 1);
+    this.projects = cfg.getStringList(SECTION_NOTE_DB, null, ONLINE_MIGRATION_PROJECTS);
   }
 
   @Override
@@ -83,22 +92,35 @@ public class OnlineNoteDbMigrator implements LifecycleListener {
     t.start();
   }
 
-  private void migrate() {
+  @VisibleForTesting
+  public void migrate() {
     logger.atInfo().log("Starting online NoteDb migration");
     if (upgradeIndex) {
       logger.atInfo().log(
           "Online index schema upgrades will be deferred until NoteDb migration is complete");
     }
     Stopwatch sw = Stopwatch.createStarted();
-    // TODO(dborowitz): maybe expose a progress monitor somewhere.
-    try (NoteDbMigrator migrator =
-        migratorBuilderProvider
-            .get()
-            .setThreads(threads)
-            .setAutoMigrate(true)
-            .setTrialMode(trial)
-            .build()) {
-      migrator.migrate();
+
+    Builder migratorBuilder =
+        migratorBuilderProvider.get().setThreads(threads).setAutoMigrate(true).setTrialMode(trial);
+
+    try {
+      // TODO(dborowitz): maybe expose a progress monitor somewhere.
+      if (projects.length > 0) {
+        try (NoteDbMigrator migrator =
+            migratorBuilder
+                .setProjects(
+                    Arrays.asList(projects).stream()
+                        .map(Project.NameKey::new)
+                        .collect(Collectors.toList()))
+                .build()) {
+          migrator.rebuild();
+        }
+      } else {
+        try (NoteDbMigrator migrator = migratorBuilder.build()) {
+          migrator.migrate();
+        }
+      }
     } catch (Exception e) {
       logger.atSevere().withCause(e).log("Error in online NoteDb migration");
     }
diff --git a/javatests/com/google/gerrit/acceptance/server/notedb/OnlineNoteDbMigrationIT.java b/javatests/com/google/gerrit/acceptance/server/notedb/OnlineNoteDbMigrationIT.java
index b9ed0f3ac7..8f0506bcf7 100644
--- a/javatests/com/google/gerrit/acceptance/server/notedb/OnlineNoteDbMigrationIT.java
+++ b/javatests/com/google/gerrit/acceptance/server/notedb/OnlineNoteDbMigrationIT.java
@@ -43,6 +43,7 @@ import com.google.gerrit.acceptance.NoHttpd;
 import com.google.gerrit.acceptance.PushOneCommit;
 import com.google.gerrit.acceptance.Sandboxed;
 import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.extensions.api.projects.ProjectInput;
 import com.google.gerrit.extensions.registration.DynamicSet;
 import com.google.gerrit.extensions.registration.RegistrationHandle;
 import com.google.gerrit.reviewdb.client.Change;
@@ -62,12 +63,14 @@ import com.google.gerrit.server.notedb.NotesMigrationState;
 import com.google.gerrit.server.notedb.rebuild.MigrationException;
 import com.google.gerrit.server.notedb.rebuild.NoteDbMigrator;
 import com.google.gerrit.server.notedb.rebuild.NotesMigrationStateListener;
+import com.google.gerrit.server.notedb.rebuild.OnlineNoteDbMigrator;
 import com.google.gerrit.server.schema.ReviewDbFactory;
 import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.NoteDbMode;
 import com.google.gwtorm.server.OrmException;
 import com.google.gwtorm.server.SchemaFactory;
 import com.google.inject.Inject;
+import com.google.inject.Injector;
 import com.google.inject.Provider;
 import java.io.IOException;
 import java.nio.file.Files;
@@ -323,6 +326,38 @@ public class OnlineNoteDbMigrationIT extends AbstractDaemonTest {
   }
 
   @Test
+  @GerritConfig(name = "noteDb.onlineMigrationProjects", value = "project2")
+  @UseLocalDisk
+  public void rebuildSubsetOfProjectsFromConfig() throws Exception {
+    Injector injector =
+        server.getTestInjector().createChildInjector(new OnlineNoteDbMigrator.Module(true));
+    OnlineNoteDbMigrator onlineNoteDbMigrator = injector.getInstance(OnlineNoteDbMigrator.class);
+
+    setNotesMigrationState(WRITE);
+
+    ProjectInput in = new ProjectInput();
+    in.name = "project2";
+    in.parent = allProjects.get();
+    in.createEmptyCommit = true;
+    gApi.projects().create(in);
+    Project.NameKey p2 = new Project.NameKey(in.name);
+
+    TestRepository<?> tr2 = cloneProject(p2, admin);
+
+    PushOneCommit.Result r1 = createChange();
+    PushOneCommit.Result r2 = pushFactory.create(db, admin.getIdent(), tr2).to("refs/for/master");
+    Change.Id id1 = r1.getChange().getId();
+    Change.Id id2 = r2.getChange().getId();
+
+    invalidateNoteDbState(id1, id2);
+
+    onlineNoteDbMigrator.migrate();
+
+    assertNotRebuilt(id1);
+    assertRebuilt(id2);
+  }
+
+  @Test
   public void rebuildNonSkippedProjects() throws Exception {
     setNotesMigrationState(WRITE);