diff options
19 files changed, 212 insertions, 49 deletions
diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt index 3594626906..6d98877225 100644 --- a/Documentation/config-gerrit.txt +++ b/Documentation/config-gerrit.txt @@ -2748,7 +2748,7 @@ by the JVM. If set to a negative value, defaults to a direct executor. [[index.batchThreads]]index.batchThreads:: + Number of threads to use for indexing in background operations, such as -online schema upgrades, and also for offline reindexing. +online schema upgrades, and also the default for offline reindexing. + If not set or set to a zero, defaults to the number of logical CPUs as returned by the JVM. If set to a negative value, defaults to a direct executor. @@ -2964,6 +2964,11 @@ Lucene documentation] for further details. + Defaults to true (throttling enabled). +During offline reindexing, setting ramBufferSize greater than the size +of index (size of specific index folder under <site_dir>/index) and +maxBufferedDocs as -1 avoids unnecessary flushes and triggers only a +single flush at the end of the process. + Sample Lucene index configuration: ---- [index] diff --git a/Documentation/pgm-reindex.txt b/Documentation/pgm-reindex.txt index 5167277b8e..5392564df3 100644 --- a/Documentation/pgm-reindex.txt +++ b/Documentation/pgm-reindex.txt @@ -20,7 +20,8 @@ Rebuilds the secondary index. == OPTIONS --threads:: - Number of threads to use for indexing. + Number of threads to use for indexing. Default is + link:config-gerrit.html#index.batchThreads[index.batchThreads] --changes-schema-version:: Schema version to reindex; default is most recent version. diff --git a/java/com/google/gerrit/acceptance/GerritServer.java b/java/com/google/gerrit/acceptance/GerritServer.java index 5e8698f15a..34f514c590 100644 --- a/java/com/google/gerrit/acceptance/GerritServer.java +++ b/java/com/google/gerrit/acceptance/GerritServer.java @@ -41,6 +41,7 @@ import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePath; import com.google.gerrit.server.git.receive.AsyncReceiveCommits; import com.google.gerrit.server.schema.JdbcAccountPatchReviewStore; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.ssh.NoSshModule; import com.google.gerrit.server.util.SocketUtil; import com.google.gerrit.server.util.SystemLog; @@ -374,7 +375,8 @@ public class GerritServer implements AutoCloseable { cfg.setString( "accountPatchReviewDb", null, "url", JdbcAccountPatchReviewStore.TEST_IN_MEMORY_URL); daemon.setEnableHttpd(desc.httpd()); - daemon.setLuceneModule(LuceneIndexModule.singleVersionAllLatest(0, isSlave(baseConfig))); + daemon.setLuceneModule( + LuceneIndexModule.singleVersionAllLatest(0, isSlave(baseConfig), AutoFlush.ENABLED)); daemon.setDatabaseForTesting( ImmutableList.of( new InMemoryTestingDatabaseModule(cfg, site, inMemoryRepoManager), diff --git a/java/com/google/gerrit/httpd/init/WebAppInitializer.java b/java/com/google/gerrit/httpd/init/WebAppInitializer.java index bf5cd2a9b8..c9dfc5817a 100644 --- a/java/com/google/gerrit/httpd/init/WebAppInitializer.java +++ b/java/com/google/gerrit/httpd/init/WebAppInitializer.java @@ -73,6 +73,7 @@ import com.google.gerrit.server.git.GarbageCollectionModule; import com.google.gerrit.server.git.GitRepositoryManagerModule; import com.google.gerrit.server.git.SearchingChangeCacheImpl; import com.google.gerrit.server.git.WorkQueue; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.IndexModule.IndexType; import com.google.gerrit.server.index.OnlineUpgrader; @@ -344,7 +345,7 @@ public class WebAppInitializer extends GuiceServletContextListener implements Fi private Module createIndexModule() { switch (indexType) { case LUCENE: - return LuceneIndexModule.latestVersion(false); + return LuceneIndexModule.latestVersion(false, AutoFlush.ENABLED); case ELASTICSEARCH: return ElasticIndexModule.latestVersion(false); default: diff --git a/java/com/google/gerrit/lucene/AbstractLuceneIndex.java b/java/com/google/gerrit/lucene/AbstractLuceneIndex.java index 7a0430c701..93c0fada6d 100644 --- a/java/com/google/gerrit/lucene/AbstractLuceneIndex.java +++ b/java/com/google/gerrit/lucene/AbstractLuceneIndex.java @@ -43,6 +43,7 @@ import com.google.gerrit.index.query.FieldBundle; import com.google.gerrit.index.query.ListResultSet; import com.google.gerrit.index.query.ResultSet; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.logging.LoggingContextAwareExecutorService; import com.google.gerrit.server.logging.LoggingContextAwareScheduledExecutorService; @@ -101,6 +102,7 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> { private final ReferenceManager<IndexSearcher> searcherManager; private final ControlledRealTimeReopenThread<IndexSearcher> reopenThread; private final Set<NrtFuture> notDoneNrtFutures; + private final AutoFlush autoFlush; private ScheduledExecutorService autoCommitExecutor; AbstractLuceneIndex( @@ -110,12 +112,14 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> { String name, String subIndex, GerritIndexWriterConfig writerConfig, - SearcherFactory searcherFactory) + SearcherFactory searcherFactory, + AutoFlush autoFlush) throws IOException { this.schema = schema; this.sitePaths = sitePaths; this.dir = dir; this.name = name; + this.autoFlush = autoFlush; String index = Joiner.on('_').skipNulls().join(name, subIndex); long commitPeriod = writerConfig.getCommitWithinMs(); @@ -209,7 +213,9 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> { } }); - reopenThread.start(); + if (autoFlush.equals(AutoFlush.ENABLED)) { + reopenThread.start(); + } } @Override @@ -459,6 +465,9 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> { } private boolean isGenAvailableNowForCurrentSearcher() { + if (autoFlush.equals(AutoFlush.DISABLED)) { + return true; + } try { return reopenThread.waitForGeneration(gen, 0); } catch (InterruptedException e) { diff --git a/java/com/google/gerrit/lucene/ChangeSubIndex.java b/java/com/google/gerrit/lucene/ChangeSubIndex.java index 98424b5e1c..8fcd0b1d04 100644 --- a/java/com/google/gerrit/lucene/ChangeSubIndex.java +++ b/java/com/google/gerrit/lucene/ChangeSubIndex.java @@ -29,6 +29,7 @@ import com.google.gerrit.index.query.Predicate; import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.reviewdb.client.Change; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.index.change.ChangeField; import com.google.gerrit.server.index.change.ChangeIndex; import com.google.gerrit.server.query.change.ChangeData; @@ -48,7 +49,8 @@ public class ChangeSubIndex extends AbstractLuceneIndex<Change.Id, ChangeData> SitePaths sitePaths, Path path, GerritIndexWriterConfig writerConfig, - SearcherFactory searcherFactory) + SearcherFactory searcherFactory, + AutoFlush autoFlush) throws IOException { this( schema, @@ -56,7 +58,8 @@ public class ChangeSubIndex extends AbstractLuceneIndex<Change.Id, ChangeData> FSDirectory.open(path), path.getFileName().toString(), writerConfig, - searcherFactory); + searcherFactory, + autoFlush); } ChangeSubIndex( @@ -65,9 +68,10 @@ public class ChangeSubIndex extends AbstractLuceneIndex<Change.Id, ChangeData> Directory dir, String subIndex, GerritIndexWriterConfig writerConfig, - SearcherFactory searcherFactory) + SearcherFactory searcherFactory, + AutoFlush autoFlush) throws IOException { - super(schema, sitePaths, dir, NAME, subIndex, writerConfig, searcherFactory); + super(schema, sitePaths, dir, NAME, subIndex, writerConfig, searcherFactory, autoFlush); } @Override diff --git a/java/com/google/gerrit/lucene/LuceneAccountIndex.java b/java/com/google/gerrit/lucene/LuceneAccountIndex.java index 0b787b6e18..272c774754 100644 --- a/java/com/google/gerrit/lucene/LuceneAccountIndex.java +++ b/java/com/google/gerrit/lucene/LuceneAccountIndex.java @@ -32,6 +32,7 @@ import com.google.gerrit.server.account.AccountCache; import com.google.gerrit.server.account.AccountState; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.account.AccountIndex; import com.google.inject.Inject; @@ -87,7 +88,8 @@ public class LuceneAccountIndex extends AbstractLuceneIndex<Account.Id, AccountS @GerritServerConfig Config cfg, SitePaths sitePaths, Provider<AccountCache> accountCache, - @Assisted Schema<AccountState> schema) + @Assisted Schema<AccountState> schema, + AutoFlush autoFlush) throws IOException { super( schema, @@ -96,7 +98,8 @@ public class LuceneAccountIndex extends AbstractLuceneIndex<Account.Id, AccountS ACCOUNTS, null, new GerritIndexWriterConfig(cfg, ACCOUNTS), - new SearcherFactory()); + new SearcherFactory(), + autoFlush); this.accountCache = accountCache; indexWriterConfig = new GerritIndexWriterConfig(cfg, ACCOUNTS); diff --git a/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/java/com/google/gerrit/lucene/LuceneChangeIndex.java index 87f9396a90..a9e431ef12 100644 --- a/java/com/google/gerrit/lucene/LuceneChangeIndex.java +++ b/java/com/google/gerrit/lucene/LuceneChangeIndex.java @@ -55,6 +55,7 @@ import com.google.gerrit.reviewdb.converter.ProtoConverter; import com.google.gerrit.server.StarredChangesUtil; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.index.IndexExecutor; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.change.ChangeField; @@ -155,7 +156,8 @@ public class LuceneChangeIndex implements ChangeIndex { SitePaths sitePaths, @IndexExecutor(INTERACTIVE) ListeningExecutorService executor, ChangeData.Factory changeDataFactory, - @Assisted Schema<ChangeData> schema) + @Assisted Schema<ChangeData> schema, + AutoFlush autoFlush) throws IOException { this.executor = executor; this.changeDataFactory = changeDataFactory; @@ -170,18 +172,35 @@ public class LuceneChangeIndex implements ChangeIndex { if (LuceneIndexModule.isInMemoryTest(cfg)) { openIndex = new ChangeSubIndex( - schema, sitePaths, new RAMDirectory(), "ramOpen", openConfig, searcherFactory); + schema, + sitePaths, + new RAMDirectory(), + "ramOpen", + openConfig, + searcherFactory, + autoFlush); closedIndex = new ChangeSubIndex( - schema, sitePaths, new RAMDirectory(), "ramClosed", closedConfig, searcherFactory); + schema, + sitePaths, + new RAMDirectory(), + "ramClosed", + closedConfig, + searcherFactory, + autoFlush); } else { Path dir = LuceneVersionManager.getDir(sitePaths, CHANGES, schema); openIndex = new ChangeSubIndex( - schema, sitePaths, dir.resolve(CHANGES_OPEN), openConfig, searcherFactory); + schema, sitePaths, dir.resolve(CHANGES_OPEN), openConfig, searcherFactory, autoFlush); closedIndex = new ChangeSubIndex( - schema, sitePaths, dir.resolve(CHANGES_CLOSED), closedConfig, searcherFactory); + schema, + sitePaths, + dir.resolve(CHANGES_CLOSED), + closedConfig, + searcherFactory, + autoFlush); } } diff --git a/java/com/google/gerrit/lucene/LuceneGroupIndex.java b/java/com/google/gerrit/lucene/LuceneGroupIndex.java index 0fdef77b96..47a5038812 100644 --- a/java/com/google/gerrit/lucene/LuceneGroupIndex.java +++ b/java/com/google/gerrit/lucene/LuceneGroupIndex.java @@ -30,6 +30,7 @@ import com.google.gerrit.server.account.GroupCache; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.group.InternalGroup; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.group.GroupIndex; import com.google.inject.Inject; @@ -83,7 +84,8 @@ public class LuceneGroupIndex extends AbstractLuceneIndex<AccountGroup.UUID, Int @GerritServerConfig Config cfg, SitePaths sitePaths, Provider<GroupCache> groupCache, - @Assisted Schema<InternalGroup> schema) + @Assisted Schema<InternalGroup> schema, + AutoFlush autoFlush) throws IOException { super( schema, @@ -92,7 +94,8 @@ public class LuceneGroupIndex extends AbstractLuceneIndex<AccountGroup.UUID, Int GROUPS, null, new GerritIndexWriterConfig(cfg, GROUPS), - new SearcherFactory()); + new SearcherFactory(), + autoFlush); this.groupCache = groupCache; indexWriterConfig = new GerritIndexWriterConfig(cfg, GROUPS); diff --git a/java/com/google/gerrit/lucene/LuceneIndexModule.java b/java/com/google/gerrit/lucene/LuceneIndexModule.java index 302a2da851..10b58c0894 100644 --- a/java/com/google/gerrit/lucene/LuceneIndexModule.java +++ b/java/com/google/gerrit/lucene/LuceneIndexModule.java @@ -19,6 +19,7 @@ import com.google.gerrit.index.IndexConfig; import com.google.gerrit.index.project.ProjectIndex; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.index.AbstractIndexModule; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.index.VersionManager; import com.google.gerrit.server.index.account.AccountIndex; import com.google.gerrit.server.index.change.ChangeIndex; @@ -28,25 +29,36 @@ import org.apache.lucene.search.BooleanQuery; import org.eclipse.jgit.lib.Config; public class LuceneIndexModule extends AbstractIndexModule { - public static LuceneIndexModule singleVersionAllLatest(int threads, boolean slave) { - return new LuceneIndexModule(ImmutableMap.of(), threads, slave); + private final AutoFlush autoFlush; + + public static LuceneIndexModule singleVersionAllLatest( + int threads, boolean slave, AutoFlush autoFlush) { + return new LuceneIndexModule(ImmutableMap.of(), threads, slave, autoFlush); } public static LuceneIndexModule singleVersionWithExplicitVersions( - Map<String, Integer> versions, int threads, boolean slave) { - return new LuceneIndexModule(versions, threads, slave); + Map<String, Integer> versions, int threads, boolean slave, AutoFlush autoFlush) { + return new LuceneIndexModule(versions, threads, slave, autoFlush); } - public static LuceneIndexModule latestVersion(boolean slave) { - return new LuceneIndexModule(null, 0, slave); + public static LuceneIndexModule latestVersion(boolean slave, AutoFlush autoFlush) { + return new LuceneIndexModule(null, 0, slave, autoFlush); } static boolean isInMemoryTest(Config cfg) { return cfg.getBoolean("index", "lucene", "testInmemory", false); } - private LuceneIndexModule(Map<String, Integer> singleVersions, int threads, boolean slave) { + private LuceneIndexModule( + Map<String, Integer> singleVersions, int threads, boolean slave, AutoFlush autoFlush) { super(singleVersions, threads, slave); + this.autoFlush = autoFlush; + } + + @Override + protected void configure() { + super.configure(); + bind(AutoFlush.class).toInstance(autoFlush); } @Override diff --git a/java/com/google/gerrit/lucene/LuceneProjectIndex.java b/java/com/google/gerrit/lucene/LuceneProjectIndex.java index 950e2060ca..601ed5fb3f 100644 --- a/java/com/google/gerrit/lucene/LuceneProjectIndex.java +++ b/java/com/google/gerrit/lucene/LuceneProjectIndex.java @@ -30,6 +30,7 @@ import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.project.ProjectCache; import com.google.gerrit.server.project.ProjectState; @@ -83,7 +84,8 @@ public class LuceneProjectIndex extends AbstractLuceneIndex<Project.NameKey, Pro @GerritServerConfig Config cfg, SitePaths sitePaths, Provider<ProjectCache> projectCache, - @Assisted Schema<ProjectData> schema) + @Assisted Schema<ProjectData> schema, + AutoFlush autoFlush) throws IOException { super( schema, @@ -92,7 +94,8 @@ public class LuceneProjectIndex extends AbstractLuceneIndex<Project.NameKey, Pro PROJECTS, null, new GerritIndexWriterConfig(cfg, PROJECTS), - new SearcherFactory()); + new SearcherFactory(), + autoFlush); this.projectCache = projectCache; indexWriterConfig = new GerritIndexWriterConfig(cfg, PROJECTS); diff --git a/java/com/google/gerrit/pgm/Daemon.java b/java/com/google/gerrit/pgm/Daemon.java index 8887385bf6..6b4a065f3f 100644 --- a/java/com/google/gerrit/pgm/Daemon.java +++ b/java/com/google/gerrit/pgm/Daemon.java @@ -82,6 +82,7 @@ import com.google.gerrit.server.git.GarbageCollectionModule; import com.google.gerrit.server.git.SearchingChangeCacheImpl; import com.google.gerrit.server.git.WorkQueue; import com.google.gerrit.server.group.PeriodicGroupIndexer; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.IndexModule.IndexType; import com.google.gerrit.server.index.OnlineUpgrader; @@ -498,7 +499,7 @@ public class Daemon extends SiteProgram { } switch (indexType) { case LUCENE: - return LuceneIndexModule.latestVersion(slave); + return LuceneIndexModule.latestVersion(slave, AutoFlush.ENABLED); case ELASTICSEARCH: return ElasticIndexModule.latestVersion(slave); default: diff --git a/java/com/google/gerrit/pgm/Reindex.java b/java/com/google/gerrit/pgm/Reindex.java index 6fd542406c..544c61d47d 100644 --- a/java/com/google/gerrit/pgm/Reindex.java +++ b/java/com/google/gerrit/pgm/Reindex.java @@ -30,6 +30,7 @@ import com.google.gerrit.pgm.util.BatchProgramModule; import com.google.gerrit.pgm.util.SiteProgram; import com.google.gerrit.server.change.ChangeResource; import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.IndexModule.IndexType; import com.google.gerrit.server.index.change.ChangeSchemaDefinitions; @@ -51,8 +52,10 @@ import org.eclipse.jgit.util.io.NullOutputStream; import org.kohsuke.args4j.Option; public class Reindex extends SiteProgram { - @Option(name = "--threads", usage = "Number of threads to use for indexing") - private int threads = Runtime.getRuntime().availableProcessors(); + @Option( + name = "--threads", + usage = "Number of threads to use for indexing. Default is index.batchThreads from config.") + private int threads = 0; @Option( name = "--changes-schema-version", @@ -149,7 +152,9 @@ public class Reindex extends SiteProgram { Module indexModule; switch (IndexModule.getIndexType(dbInjector)) { case LUCENE: - indexModule = LuceneIndexModule.singleVersionWithExplicitVersions(versions, threads, slave); + indexModule = + LuceneIndexModule.singleVersionWithExplicitVersions( + versions, threads, slave, AutoFlush.DISABLED); break; case ELASTICSEARCH: indexModule = diff --git a/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java b/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java index 12a44dc535..54618a19e5 100644 --- a/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java +++ b/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java @@ -17,6 +17,7 @@ package com.google.gerrit.pgm.init.index.lucene; import com.google.gerrit.lucene.LuceneAccountIndex; import com.google.gerrit.lucene.LuceneGroupIndex; import com.google.gerrit.pgm.init.index.IndexModuleOnInit; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.index.account.AccountIndex; import com.google.gerrit.server.index.group.GroupIndex; import com.google.inject.AbstractModule; @@ -36,5 +37,7 @@ public class LuceneIndexModuleOnInit extends AbstractModule { .build(GroupIndex.Factory.class)); install(new IndexModuleOnInit()); + + bind(AutoFlush.class).toInstance(AutoFlush.DISABLED); } } diff --git a/java/com/google/gerrit/server/git/meta/VersionedMetaData.java b/java/com/google/gerrit/server/git/meta/VersionedMetaData.java index 86942998a6..cce81b0036 100644 --- a/java/com/google/gerrit/server/git/meta/VersionedMetaData.java +++ b/java/com/google/gerrit/server/git/meta/VersionedMetaData.java @@ -213,6 +213,27 @@ public abstract class VersionedMetaData { } /** + * Update this metadata branch, recording a new commit on its reference. This method mutates its + * receiver. + * + * @param update helper information to define the update that will occur. + * @param objInserter Shared object inserter. + * @param objReader Shared object reader. + * @param revWalk Shared rev walk. + * @return the commit that was created + * @throws IOException if there is a storage problem and the update cannot be executed as + * requested or if it failed because of a concurrent update to the same reference + */ + public RevCommit commit( + MetaDataUpdate update, ObjectInserter objInserter, ObjectReader objReader, RevWalk revWalk) + throws IOException { + try (BatchMetaDataUpdate batch = openUpdate(update, objInserter, objReader, revWalk)) { + batch.write(update.getCommitBuilder()); + return batch.commit(); + } + } + + /** * Creates a new commit and a new ref based on this commit. This method mutates its receiver. * * @param update helper information to define the update that will occur. @@ -259,11 +280,39 @@ public abstract class VersionedMetaData { * @throws IOException if the update failed. */ public BatchMetaDataUpdate openUpdate(MetaDataUpdate update) throws IOException { + return openUpdate(update, null, null, null); + } + + /** + * Open a batch of updates to the same metadata ref. + * + * <p>This allows making multiple commits to a single metadata ref, at the end of which is a + * single ref update. For batching together updates to multiple refs (each consisting of one or + * more commits against their respective refs), create the {@link MetaDataUpdate} with a {@link + * BatchRefUpdate}. + * + * <p>A ref update produced by this {@link BatchMetaDataUpdate} is only committed if there is no + * associated {@link BatchRefUpdate}. As a result, the configured ref updated event is not fired + * if there is an associated batch. + * + * <p>If object inserter, reader and revwalk are provided, then the updates are not flushed, + * allowing callers the flexibility to flush only once after several updates. + * + * @param update helper info about the update. + * @param objInserter Shared object inserter. + * @param objReader Shared object reader. + * @param revWalk Shared rev walk. + * @throws IOException if the update failed. + */ + public BatchMetaDataUpdate openUpdate( + MetaDataUpdate update, ObjectInserter objInserter, ObjectReader objReader, RevWalk revWalk) + throws IOException { final Repository db = update.getRepository(); - inserter = db.newObjectInserter(); - reader = inserter.newReader(); - final RevWalk rw = new RevWalk(reader); + inserter = objInserter == null ? db.newObjectInserter() : objInserter; + reader = objReader == null ? inserter.newReader() : objReader; + final RevWalk rw = revWalk == null ? new RevWalk(reader) : revWalk; + final RevTree tree = revision != null ? rw.parseTree(revision) : null; newTree = readTree(tree); return new BatchMetaDataUpdate() { @@ -376,7 +425,7 @@ public abstract class VersionedMetaData { newTree = null; rw.close(); - if (inserter != null) { + if (objInserter == null && inserter != null) { inserter.close(); inserter = null; } @@ -392,7 +441,9 @@ public abstract class VersionedMetaData { BatchRefUpdate bru = update.getBatch(); if (bru != null) { bru.addCommand(new ReceiveCommand(oldId.toObjectId(), newId.toObjectId(), refName)); - inserter.flush(); + if (objInserter == null) { + inserter.flush(); + } revision = rw.parseCommit(newId); return revision; } diff --git a/java/com/google/gerrit/server/index/AutoFlush.java b/java/com/google/gerrit/server/index/AutoFlush.java new file mode 100644 index 0000000000..3d131105dd --- /dev/null +++ b/java/com/google/gerrit/server/index/AutoFlush.java @@ -0,0 +1,20 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index; + +public enum AutoFlush { + ENABLED, + DISABLED +} diff --git a/java/com/google/gerrit/server/index/IndexModule.java b/java/com/google/gerrit/server/index/IndexModule.java index e7b892d611..612c637dfa 100644 --- a/java/com/google/gerrit/server/index/IndexModule.java +++ b/java/com/google/gerrit/server/index/IndexModule.java @@ -224,13 +224,14 @@ public class IndexModule extends LifecycleModule { return interactiveExecutor; } int threads = this.threads; - if (threads < 0) { - return MoreExecutors.newDirectExecutorService(); - } else if (threads == 0) { + if (threads == 0) { threads = config.getInt( "index", null, "threads", Runtime.getRuntime().availableProcessors() / 2 + 1); } + if (threads < 0) { + return MoreExecutors.newDirectExecutorService(); + } return MoreExecutors.listeningDecorator( workQueue.createQueue(threads, "Index-Interactive", true)); } @@ -243,11 +244,13 @@ public class IndexModule extends LifecycleModule { if (batchExecutor != null) { return batchExecutor; } - int threads = config.getInt("index", null, "batchThreads", 0); + int threads = this.threads; + if (threads == 0) { + threads = + config.getInt("index", null, "batchThreads", Runtime.getRuntime().availableProcessors()); + } if (threads < 0) { return MoreExecutors.newDirectExecutorService(); - } else if (threads == 0) { - threads = Runtime.getRuntime().availableProcessors(); } return MoreExecutors.listeningDecorator(workQueue.createQueue(threads, "Index-Batch", true)); } diff --git a/java/com/google/gerrit/server/mail/send/OutgoingEmail.java b/java/com/google/gerrit/server/mail/send/OutgoingEmail.java index 3bb710bb02..97035f23b4 100644 --- a/java/com/google/gerrit/server/mail/send/OutgoingEmail.java +++ b/java/com/google/gerrit/server/mail/send/OutgoingEmail.java @@ -21,6 +21,7 @@ import static java.util.Objects.requireNonNull; import com.google.common.collect.Sets; import com.google.common.flogger.FluentLogger; import com.google.gerrit.exceptions.EmailException; +import com.google.gerrit.common.Nullable; import com.google.gerrit.extensions.api.changes.RecipientType; import com.google.gerrit.extensions.client.GeneralPreferencesInfo; import com.google.gerrit.extensions.client.GeneralPreferencesInfo.EmailFormat; @@ -333,7 +334,7 @@ public abstract class OutgoingEmail { } /** Lookup a human readable name for an account, usually the "full name". */ - protected String getNameFor(Account.Id accountId) { + protected String getNameFor(@Nullable Account.Id accountId) { if (accountId == null) { return args.gerritPersonIdent.getName(); } @@ -359,7 +360,14 @@ public abstract class OutgoingEmail { * @param accountId user to fetch. * @return name/email of account, or Anonymous Coward if unset. */ - protected String getNameEmailFor(Account.Id accountId) { + protected String getNameEmailFor(@Nullable Account.Id accountId) { + if (accountId == null) { + return args.gerritPersonIdent.getName() + + " <" + + args.gerritPersonIdent.getEmailAddress() + + ">"; + } + Optional<Account> account = args.accountCache.get(accountId).map(AccountState::getAccount); if (account.isPresent()) { String name = account.get().getFullName(); @@ -380,9 +388,13 @@ public abstract class OutgoingEmail { * username. If no username is set, this function returns null. * * @param accountId user to fetch. - * @return name/email of account, username, or null if unset. + * @return name/email of account, username, or null if unset or the accountId is null. */ - protected String getUserNameEmailFor(Account.Id accountId) { + protected String getUserNameEmailFor(@Nullable Account.Id accountId) { + if (accountId == null) { + return null; + } + Optional<AccountState> accountState = args.accountCache.get(accountId); if (!accountState.isPresent()) { return null; diff --git a/java/com/google/gerrit/testing/InMemoryModule.java b/java/com/google/gerrit/testing/InMemoryModule.java index 83b9e2b6e0..9d07780af4 100644 --- a/java/com/google/gerrit/testing/InMemoryModule.java +++ b/java/com/google/gerrit/testing/InMemoryModule.java @@ -61,6 +61,7 @@ import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.git.PerThreadRequestScope; import com.google.gerrit.server.git.SearchingChangeCacheImpl; import com.google.gerrit.server.git.WorkQueue; +import com.google.gerrit.server.index.AutoFlush; import com.google.gerrit.server.index.IndexModule.IndexType; import com.google.gerrit.server.index.account.AccountSchemaDefinitions; import com.google.gerrit.server.index.account.AllAccountsIndexer; @@ -304,8 +305,13 @@ public class InMemoryModule extends FactoryModule { boolean slave = cfg.getBoolean("container", "slave", false); Class<?> clazz = Class.forName(moduleClassName); Method m = - clazz.getMethod("singleVersionWithExplicitVersions", Map.class, int.class, boolean.class); - return (Module) m.invoke(null, getSingleSchemaVersions(), 0, slave); + clazz.getMethod( + "singleVersionWithExplicitVersions", + Map.class, + int.class, + boolean.class, + AutoFlush.class); + return (Module) m.invoke(null, getSingleSchemaVersions(), 0, slave, AutoFlush.ENABLED); } catch (ClassNotFoundException | SecurityException | NoSuchMethodException |