Allow to configure projects to be skipped during reindexing

Some projects may cause reindexing to be very slow, but might not
be immediately important, e.g. because they are archived/read-only.

To have a mostly ready index faster, this change allows to configure
a list of projects that will be skipped during reindexing. These
projects can then separately be reindexed later using the
'gerrit index changes-in-project' command.

Release-Notes: Option to skip projects during change reindexing
Change-Id: I3ed63244607cb02c16571dd898f3cc6fa97c9166
(cherry picked from commit 0030ff41100604287d0fdb28fd2abd57fdece6f9)

2 files changed, 23 insertions, 3 deletions

diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt
index 7f2550900f..f795defe15 100644
--- a/Documentation/config-gerrit.txt
+++ b/Documentation/config-gerrit.txt
@@ -3420,6 +3420,15 @@ advantage of new search features without restarting the server.
 +
 Defaults to true.
 
+[[index.excludeProjectFromChangeReindex]]index.excludeProjectFromChangeReindex::
++
+A list of projects that will be excluded from reindexing. This can be used
+to exclude projects which are expensive to reindex to prioritize the other
+projects.
++
+Excluded projects can later be reindexed by for example using the
+link:cmd-index-changes-in-project.html[index changes in project command].
+
 [[index.paginationType]]index.paginationType::
 +
 The pagination type to use when index queries are repeated to
diff --git a/java/com/google/gerrit/server/index/change/AllChangesIndexer.java b/java/com/google/gerrit/server/index/change/AllChangesIndexer.java
index 4f411a2ff7..86326dc715 100644
--- a/java/com/google/gerrit/server/index/change/AllChangesIndexer.java
+++ b/java/com/google/gerrit/server/index/change/AllChangesIndexer.java
@@ -22,7 +22,7 @@ import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
 import com.google.auto.value.AutoValue;
 import com.google.common.base.Stopwatch;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Sets;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
@@ -31,6 +31,7 @@ import com.google.gerrit.common.Nullable;
 import com.google.gerrit.entities.Change;
 import com.google.gerrit.entities.Project;
 import com.google.gerrit.index.SiteIndexer;
+import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.git.GitRepositoryManager;
 import com.google.gerrit.server.git.MultiProgressMonitor;
 import com.google.gerrit.server.git.MultiProgressMonitor.Task;
@@ -46,10 +47,13 @@ import com.google.inject.Inject;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import org.eclipse.jgit.lib.Config;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ProgressMonitor;
 import org.eclipse.jgit.lib.Repository;
@@ -82,6 +86,7 @@ public class AllChangesIndexer extends SiteIndexer<Change.Id, ChangeData, Change
   private final ChangeIndexer.Factory indexerFactory;
   private final ChangeNotes.Factory notesFactory;
   private final ProjectCache projectCache;
+  private final Set<Project.NameKey> projectsToSkip;
 
   @Inject
   AllChangesIndexer(
@@ -91,7 +96,8 @@ public class AllChangesIndexer extends SiteIndexer<Change.Id, ChangeData, Change
       @IndexExecutor(BATCH) ListeningExecutorService executor,
       ChangeIndexer.Factory indexerFactory,
       ChangeNotes.Factory notesFactory,
-      ProjectCache projectCache) {
+      ProjectCache projectCache,
+      @GerritServerConfig Config config) {
     this.multiProgressMonitorFactory = multiProgressMonitorFactory;
     this.changeDataFactory = changeDataFactory;
     this.repoManager = repoManager;
@@ -99,6 +105,11 @@ public class AllChangesIndexer extends SiteIndexer<Change.Id, ChangeData, Change
     this.indexerFactory = indexerFactory;
     this.notesFactory = notesFactory;
     this.projectCache = projectCache;
+    this.projectsToSkip =
+        Sets.newHashSet(config.getStringList("index", null, "excludeProjectFromChangeReindex"))
+            .stream()
+            .map(p -> Project.NameKey.parse(p))
+            .collect(Collectors.toSet());
   }
 
   @AutoValue
@@ -302,7 +313,7 @@ public class AllChangesIndexer extends SiteIndexer<Change.Id, ChangeData, Change
     }
 
     private List<ListenableFuture<?>> schedule() throws ProjectsCollectionFailure {
-      ImmutableSortedSet<Project.NameKey> projects = projectCache.all();
+      Set<Project.NameKey> projects = Sets.difference(projectCache.all(), projectsToSkip);
       int projectCount = projects.size();
       slicingProjects = mpm.beginSubTask("Slicing projects", projectCount);
       for (Project.NameKey name : projects) {