Avoid auto-reindex of projects during init when unneeded

The forced reindex is not needed when the project index is present.

Preserve the automatic reindex of projects for new sites and defer
to reindex site program or online reindexing if project index already
exists.

The check is implemented in index backend agnostic manner, so that it
should work for both supported backends: Lucene and Elasticsearch. It
should also work independently whether or not a project index schema
migration is needed during upgrade.

Reindexing projects is a very expensive operation and can turn a simple
upgrade to a long and painful operation because of the increase of the
migration time and the amount of memory needed.

Bug: Issue 12680
Change-Id: I3a4f1d07405f7bb631467d1f005d48cb56ee867f

3 files changed, 67 insertions, 8 deletions

diff --git a/java/com/google/gerrit/pgm/Init.java b/java/com/google/gerrit/pgm/Init.java
index 799377c8a8..3593d8abca 100644
--- a/java/com/google/gerrit/pgm/Init.java
+++ b/java/com/google/gerrit/pgm/Init.java
@@ -30,6 +30,7 @@ import com.google.gerrit.pgm.init.api.ConsoleUI;
 import com.google.gerrit.pgm.util.ErrorLogFile;
 import com.google.gerrit.server.config.GerritServerConfigModule;
 import com.google.gerrit.server.config.SitePath;
+import com.google.gerrit.server.index.GerritIndexStatus;
 import com.google.gerrit.server.ioutil.HostPlatform;
 import com.google.gerrit.server.securestore.SecureStoreClassName;
 import com.google.gerrit.server.util.ReplicaUtil;
@@ -60,9 +61,6 @@ public class Init extends BaseInit {
   @Option(name = "--no-auto-start", usage = "Don't automatically start daemon after init")
   private boolean noAutoStart;
 
-  @Option(name = "--no-reindex", usage = "Don't automatically reindex any entities")
-  private boolean noReindex;
-
   @Option(name = "--skip-plugins", usage = "Don't install plugins")
   private boolean skipPlugins;
 
@@ -91,6 +89,8 @@ public class Init extends BaseInit {
 
   @Inject Browser browser;
 
+  private boolean projectsIndexExists;
+
   public Init() {
     super(new WarDistribution(), null);
   }
@@ -103,6 +103,7 @@ public class Init extends BaseInit {
 
   @Override
   protected boolean beforeInit(SiteInit init) throws Exception {
+    projectsIndexExists = new GerritIndexStatus(init.site).exists(ProjectSchemaDefinitions.NAME);
     ErrorLogFile.errorOnlyConsole();
 
     if (!skipPlugins) {
@@ -145,7 +146,7 @@ public class Init extends BaseInit {
         });
     modules.add(new GerritServerConfigModule());
     Guice.createInjector(modules).injectMembers(this);
-    if (!ReplicaUtil.isReplica(run.flags.cfg)) {
+    if (!ReplicaUtil.isReplica(run.flags.cfg) && !projectsIndexExists) {
       reindexProjects();
     }
     start(run);
@@ -260,9 +261,6 @@ public class Init extends BaseInit {
   }
 
   private void reindexProjects() throws Exception {
-    if (noReindex) {
-      return;
-    }
     // Reindex all projects, so that we bootstrap the project index for new installations
     List<String> reindexArgs =
         ImmutableList.of(
diff --git a/java/com/google/gerrit/server/index/GerritIndexStatus.java b/java/com/google/gerrit/server/index/GerritIndexStatus.java
index 6d59100911..9f0622e9d6 100644
--- a/java/com/google/gerrit/server/index/GerritIndexStatus.java
+++ b/java/com/google/gerrit/server/index/GerritIndexStatus.java
@@ -48,6 +48,10 @@ public class GerritIndexStatus {
     return cfg.getBoolean(SECTION, indexDirName(indexName, version), KEY_READY, false);
   }
 
+  public boolean exists(String indexName) {
+    return cfg.getSubsections(SECTION).stream().anyMatch(n -> n.startsWith(indexName));
+  }
+
   public void save() throws IOException {
     cfg.save();
   }
diff --git a/javatests/com/google/gerrit/acceptance/pgm/InitIT.java b/javatests/com/google/gerrit/acceptance/pgm/InitIT.java
index e48088e43c..4caee6482c 100644
--- a/javatests/com/google/gerrit/acceptance/pgm/InitIT.java
+++ b/javatests/com/google/gerrit/acceptance/pgm/InitIT.java
@@ -26,7 +26,16 @@ import com.google.gerrit.index.project.ProjectData;
 import com.google.gerrit.index.project.ProjectIndexCollection;
 import com.google.gerrit.server.config.AllProjectsName;
 import com.google.gerrit.server.config.AllUsersName;
+import java.io.IOException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Comparator;
 import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.eclipse.jgit.util.FS;
 import org.junit.Test;
 
 @NoHttpd
@@ -34,7 +43,7 @@ public class InitIT extends StandaloneSiteTest {
 
   @Test
   public void indexesAllProjectsAndAllUsers() throws Exception {
-    runGerrit("init", "-d", sitePaths.site_path.toString(), "--show-stack-trace");
+    initSite();
     try (ServerContext ctx = startServer()) {
       ProjectIndexCollection projectIndex =
           ctx.getInjector().getInstance(ProjectIndexCollection.class);
@@ -48,4 +57,52 @@ public class InitIT extends StandaloneSiteTest {
       assertThat(allUsersData).isPresent();
     }
   }
+
+  @Test
+  public void initDoesNotReindexProjectsOnExistingSites() throws Exception {
+    initSite();
+
+    // Simulate a projects indexes files modified in the past by 3 seconds
+    Optional<Instant> projectsLastModified =
+        getProjectsIndexLastModified(sitePaths.index_dir).map(t -> t.minusSeconds(3));
+    assertThat(projectsLastModified).isPresent();
+    setProjectsIndexLastModifiedInThePast(sitePaths.index_dir, projectsLastModified.get());
+
+    initSite();
+    Optional<Instant> projectsLastModifiedAfterInit =
+        getProjectsIndexLastModified(sitePaths.index_dir);
+
+    // Verify that projects index files haven't been updated
+    assertThat(projectsLastModified).isEqualTo(projectsLastModifiedAfterInit);
+  }
+
+  private void initSite() throws Exception {
+    runGerrit("init", "-d", sitePaths.site_path.toString(), "--show-stack-trace");
+  }
+
+  private void setProjectsIndexLastModifiedInThePast(Path indexDir, Instant time)
+      throws IOException {
+    for (Path path : getAllProjectsIndexFiles(indexDir).collect(Collectors.toList())) {
+      FS.DETECTED.setLastModified(path, time);
+    }
+  }
+
+  private Optional<Instant> getProjectsIndexLastModified(Path indexDir) throws IOException {
+    return getAllProjectsIndexFiles(indexDir)
+        .map(FS.DETECTED::lastModifiedInstant)
+        .max(Comparator.comparingLong(Instant::toEpochMilli));
+  }
+
+  private Stream<Path> getAllProjectsIndexFiles(Path indexDir) throws IOException {
+    Optional<Path> projectsPath =
+        Files.walk(indexDir, 1)
+            .filter(Files::isDirectory)
+            .filter(p -> p.getFileName().toString().startsWith("projects_"))
+            .findFirst();
+    if (!projectsPath.isPresent()) {
+      return Stream.empty();
+    }
+
+    return Files.walk(projectsPath.get(), 1, FileVisitOption.FOLLOW_LINKS);
+  }
 }