diff options
author | Nasser Grainawi <nasser@codeaurora.org> | 2021-11-17 05:08:28 +0000 |
---|---|---|
committer | Gerrit Code Review <noreply-gerritcodereview@google.com> | 2021-11-17 05:08:28 +0000 |
commit | e6e6a31de0fad6069ea010718df9d54d7ba4672b (patch) | |
tree | bc3013082c360ebc7e655bf218a6faa8f0f5e7f7 | |
parent | 9dd8e1373f7cbce272281185eb0d868f0c9937b4 (diff) | |
parent | 17f1e5be123abdea2ba7b94b37c265dcf44b6fca (diff) |
Merge "Merge branch 'stable-3.2' into stable-3.3" into stable-3.3
29 files changed, 325 insertions, 84 deletions
diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt index 750a002d30..95078c18a4 100644 --- a/Documentation/config-gerrit.txt +++ b/Documentation/config-gerrit.txt @@ -3054,7 +3054,7 @@ by the JVM. If set to a negative value, defaults to a direct executor. [[index.batchThreads]]index.batchThreads:: + Number of threads to use for indexing in background operations, such as -online schema upgrades, and also for offline reindexing. +online schema upgrades, and also the default for offline reindexing. + If not set or set to a zero, defaults to the number of logical CPUs as returned by the JVM. If set to a negative value, defaults to a direct executor. @@ -3257,6 +3257,11 @@ Lucene documentation,role=external,window=_blank] for further details. + Defaults to true (throttling enabled). +During offline reindexing, setting ramBufferSize greater than the size +of index (size of specific index folder under <site_dir>/index) and +maxBufferedDocs as -1 avoids unnecessary flushes and triggers only a +single flush at the end of the process. + Sample Lucene index configuration: ---- [index] diff --git a/Documentation/pgm-reindex.txt b/Documentation/pgm-reindex.txt index 5167277b8e..5392564df3 100644 --- a/Documentation/pgm-reindex.txt +++ b/Documentation/pgm-reindex.txt @@ -20,7 +20,8 @@ Rebuilds the secondary index. == OPTIONS --threads:: - Number of threads to use for indexing. + Number of threads to use for indexing. Default is + link:config-gerrit.html#index.batchThreads[index.batchThreads] --changes-schema-version:: Schema version to reindex; default is most recent version. diff --git a/java/com/google/gerrit/acceptance/GerritServer.java b/java/com/google/gerrit/acceptance/GerritServer.java index 3f8a5a82cd..cfc0ce472d 100644 --- a/java/com/google/gerrit/acceptance/GerritServer.java +++ b/java/com/google/gerrit/acceptance/GerritServer.java @@ -51,6 +51,7 @@ import com.google.gerrit.server.config.GerritRuntime; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePath; import com.google.gerrit.server.git.receive.AsyncReceiveCommits; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.gerrit.server.schema.JdbcAccountPatchReviewStore; import com.google.gerrit.server.ssh.NoSshModule; import com.google.gerrit.server.util.ReplicaUtil; @@ -428,7 +429,8 @@ public class GerritServer implements AutoCloseable { "accountPatchReviewDb", null, "url", JdbcAccountPatchReviewStore.TEST_IN_MEMORY_URL); daemon.setEnableHttpd(desc.httpd()); daemon.setLuceneModule( - LuceneIndexModule.singleVersionAllLatest(0, ReplicaUtil.isReplica(baseConfig))); + LuceneIndexModule.singleVersionAllLatest( + 0, ReplicaUtil.isReplica(baseConfig), AutoFlush.ENABLED)); daemon.setDatabaseForTesting( ImmutableList.of( new InMemoryTestingDatabaseModule(cfg, site, inMemoryRepoManager), diff --git a/java/com/google/gerrit/httpd/init/WebAppInitializer.java b/java/com/google/gerrit/httpd/init/WebAppInitializer.java index 193c4f1486..e4a86f5c0a 100644 --- a/java/com/google/gerrit/httpd/init/WebAppInitializer.java +++ b/java/com/google/gerrit/httpd/init/WebAppInitializer.java @@ -79,6 +79,7 @@ import com.google.gerrit.server.git.WorkQueue; import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.OnlineUpgrader; import com.google.gerrit.server.index.VersionManager; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.gerrit.server.mail.SignedTokenEmailTokenVerifier; import com.google.gerrit.server.mail.receive.MailReceiver; import com.google.gerrit.server.mail.send.SmtpEmailSender; @@ -357,7 +358,7 @@ public class WebAppInitializer extends GuiceServletContextListener implements Fi private Module createIndexModule() { if (indexType.isLucene()) { - return LuceneIndexModule.latestVersion(false); + return LuceneIndexModule.latestVersion(false, AutoFlush.ENABLED); } else if (indexType.isElasticsearch()) { return ElasticIndexModule.latestVersion(false); } else { diff --git a/java/com/google/gerrit/lucene/AbstractLuceneIndex.java b/java/com/google/gerrit/lucene/AbstractLuceneIndex.java index 5392ab464a..1e6ccac89a 100644 --- a/java/com/google/gerrit/lucene/AbstractLuceneIndex.java +++ b/java/com/google/gerrit/lucene/AbstractLuceneIndex.java @@ -45,6 +45,7 @@ import com.google.gerrit.index.query.ListResultSet; import com.google.gerrit.index.query.ResultSet; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.index.IndexUtils; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.gerrit.server.logging.LoggingContextAwareExecutorService; import com.google.gerrit.server.logging.LoggingContextAwareScheduledExecutorService; import java.io.IOException; @@ -105,6 +106,7 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> { private final ReferenceManager<IndexSearcher> searcherManager; private final ControlledRealTimeReopenThread<IndexSearcher> reopenThread; private final Set<NrtFuture> notDoneNrtFutures; + private final AutoFlush autoFlush; private ScheduledExecutorService autoCommitExecutor; AbstractLuceneIndex( @@ -115,13 +117,15 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> { ImmutableSet<String> skipFields, String subIndex, GerritIndexWriterConfig writerConfig, - SearcherFactory searcherFactory) + SearcherFactory searcherFactory, + AutoFlush autoFlush) throws IOException { this.schema = schema; this.sitePaths = sitePaths; this.dir = dir; this.name = name; this.skipFields = skipFields; + this.autoFlush = autoFlush; String index = Joiner.on('_').skipNulls().join(name, subIndex); long commitPeriod = writerConfig.getCommitWithinMs(); @@ -215,7 +219,9 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> { } }); - reopenThread.start(); + if (autoFlush.equals(AutoFlush.ENABLED)) { + reopenThread.start(); + } } @Override @@ -484,6 +490,9 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> { } private boolean isGenAvailableNowForCurrentSearcher() { + if (autoFlush.equals(AutoFlush.DISABLED)) { + return true; + } try { return reopenThread.waitForGeneration(gen, 0); } catch (InterruptedException e) { diff --git a/java/com/google/gerrit/lucene/ChangeSubIndex.java b/java/com/google/gerrit/lucene/ChangeSubIndex.java index e51a91a7e6..5c24b9a009 100644 --- a/java/com/google/gerrit/lucene/ChangeSubIndex.java +++ b/java/com/google/gerrit/lucene/ChangeSubIndex.java @@ -33,6 +33,7 @@ import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.index.change.ChangeField; import com.google.gerrit.server.index.change.ChangeIndex; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.gerrit.server.query.change.ChangeData; import java.io.IOException; import java.nio.file.Path; @@ -51,7 +52,8 @@ public class ChangeSubIndex extends AbstractLuceneIndex<Change.Id, ChangeData> Path path, ImmutableSet<String> skipFields, GerritIndexWriterConfig writerConfig, - SearcherFactory searcherFactory) + SearcherFactory searcherFactory, + AutoFlush autoFlush) throws IOException { this( schema, @@ -60,7 +62,8 @@ public class ChangeSubIndex extends AbstractLuceneIndex<Change.Id, ChangeData> path.getFileName().toString(), skipFields, writerConfig, - searcherFactory); + searcherFactory, + autoFlush); } ChangeSubIndex( @@ -70,9 +73,19 @@ public class ChangeSubIndex extends AbstractLuceneIndex<Change.Id, ChangeData> String subIndex, ImmutableSet<String> skipFields, GerritIndexWriterConfig writerConfig, - SearcherFactory searcherFactory) + SearcherFactory searcherFactory, + AutoFlush autoFlush) throws IOException { - super(schema, sitePaths, dir, NAME, skipFields, subIndex, writerConfig, searcherFactory); + super( + schema, + sitePaths, + dir, + NAME, + skipFields, + subIndex, + writerConfig, + searcherFactory, + autoFlush); } @Override diff --git a/java/com/google/gerrit/lucene/LuceneAccountIndex.java b/java/com/google/gerrit/lucene/LuceneAccountIndex.java index 242cffdbb0..87b7ccef6b 100644 --- a/java/com/google/gerrit/lucene/LuceneAccountIndex.java +++ b/java/com/google/gerrit/lucene/LuceneAccountIndex.java @@ -36,6 +36,7 @@ import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.account.AccountIndex; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.assistedinject.Assisted; @@ -94,7 +95,8 @@ public class LuceneAccountIndex extends AbstractLuceneIndex<Account.Id, AccountS @GerritServerConfig Config cfg, SitePaths sitePaths, Provider<AccountCache> accountCache, - @Assisted Schema<AccountState> schema) + @Assisted Schema<AccountState> schema, + AutoFlush autoFlush) throws IOException { super( schema, @@ -104,7 +106,8 @@ public class LuceneAccountIndex extends AbstractLuceneIndex<Account.Id, AccountS ImmutableSet.of(), null, new GerritIndexWriterConfig(cfg, ACCOUNTS), - new SearcherFactory()); + new SearcherFactory(), + autoFlush); this.accountCache = accountCache; indexWriterConfig = new GerritIndexWriterConfig(cfg, ACCOUNTS); diff --git a/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/java/com/google/gerrit/lucene/LuceneChangeIndex.java index bf1a16645f..bb97936bad 100644 --- a/java/com/google/gerrit/lucene/LuceneChangeIndex.java +++ b/java/com/google/gerrit/lucene/LuceneChangeIndex.java @@ -64,6 +64,7 @@ import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.change.ChangeField; import com.google.gerrit.server.index.change.ChangeIndex; import com.google.gerrit.server.index.change.ChangeIndexRewriter; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.gerrit.server.project.SubmitRuleOptions; import com.google.gerrit.server.query.change.ChangeData; import com.google.gerrit.server.query.change.ChangeDataSource; @@ -179,7 +180,8 @@ public class LuceneChangeIndex implements ChangeIndex { SitePaths sitePaths, @IndexExecutor(INTERACTIVE) ListeningExecutorService executor, ChangeData.Factory changeDataFactory, - @Assisted Schema<ChangeData> schema) + @Assisted Schema<ChangeData> schema, + AutoFlush autoFlush) throws IOException { this.executor = executor; this.changeDataFactory = changeDataFactory; @@ -204,7 +206,8 @@ public class LuceneChangeIndex implements ChangeIndex { "ramOpen", skipFields, openConfig, - searcherFactory); + searcherFactory, + autoFlush); closedIndex = new ChangeSubIndex( schema, @@ -213,7 +216,8 @@ public class LuceneChangeIndex implements ChangeIndex { "ramClosed", skipFields, closedConfig, - searcherFactory); + searcherFactory, + autoFlush); } else { Path dir = LuceneVersionManager.getDir(sitePaths, CHANGES, schema); openIndex = @@ -223,7 +227,8 @@ public class LuceneChangeIndex implements ChangeIndex { dir.resolve(CHANGES_OPEN), skipFields, openConfig, - searcherFactory); + searcherFactory, + autoFlush); closedIndex = new ChangeSubIndex( schema, @@ -231,7 +236,8 @@ public class LuceneChangeIndex implements ChangeIndex { dir.resolve(CHANGES_CLOSED), skipFields, closedConfig, - searcherFactory); + searcherFactory, + autoFlush); } idField = this.schema.useLegacyNumericFields() ? LEGACY_ID : LEGACY_ID_STR; diff --git a/java/com/google/gerrit/lucene/LuceneGroupIndex.java b/java/com/google/gerrit/lucene/LuceneGroupIndex.java index 3d1d471768..c1b44c7829 100644 --- a/java/com/google/gerrit/lucene/LuceneGroupIndex.java +++ b/java/com/google/gerrit/lucene/LuceneGroupIndex.java @@ -33,6 +33,7 @@ import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.group.InternalGroup; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.group.GroupIndex; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.assistedinject.Assisted; @@ -84,7 +85,8 @@ public class LuceneGroupIndex extends AbstractLuceneIndex<AccountGroup.UUID, Int @GerritServerConfig Config cfg, SitePaths sitePaths, Provider<GroupCache> groupCache, - @Assisted Schema<InternalGroup> schema) + @Assisted Schema<InternalGroup> schema, + AutoFlush autoFlush) throws IOException { super( schema, @@ -94,7 +96,8 @@ public class LuceneGroupIndex extends AbstractLuceneIndex<AccountGroup.UUID, Int ImmutableSet.of(), null, new GerritIndexWriterConfig(cfg, GROUPS), - new SearcherFactory()); + new SearcherFactory(), + autoFlush); this.groupCache = groupCache; indexWriterConfig = new GerritIndexWriterConfig(cfg, GROUPS); diff --git a/java/com/google/gerrit/lucene/LuceneIndexModule.java b/java/com/google/gerrit/lucene/LuceneIndexModule.java index 302a2da851..73256b1171 100644 --- a/java/com/google/gerrit/lucene/LuceneIndexModule.java +++ b/java/com/google/gerrit/lucene/LuceneIndexModule.java @@ -14,6 +14,7 @@ package com.google.gerrit.lucene; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.google.gerrit.index.IndexConfig; import com.google.gerrit.index.project.ProjectIndex; @@ -23,30 +24,48 @@ import com.google.gerrit.server.index.VersionManager; import com.google.gerrit.server.index.account.AccountIndex; import com.google.gerrit.server.index.change.ChangeIndex; import com.google.gerrit.server.index.group.GroupIndex; +import com.google.gerrit.server.index.options.AutoFlush; import java.util.Map; import org.apache.lucene.search.BooleanQuery; import org.eclipse.jgit.lib.Config; public class LuceneIndexModule extends AbstractIndexModule { - public static LuceneIndexModule singleVersionAllLatest(int threads, boolean slave) { - return new LuceneIndexModule(ImmutableMap.of(), threads, slave); + private final AutoFlush autoFlush; + + public static LuceneIndexModule singleVersionAllLatest( + int threads, boolean slave, AutoFlush autoFlush) { + return new LuceneIndexModule(ImmutableMap.of(), threads, slave, autoFlush); } + @VisibleForTesting public static LuceneIndexModule singleVersionWithExplicitVersions( Map<String, Integer> versions, int threads, boolean slave) { - return new LuceneIndexModule(versions, threads, slave); + return new LuceneIndexModule(versions, threads, slave, AutoFlush.ENABLED); + } + + public static LuceneIndexModule singleVersionWithExplicitVersions( + Map<String, Integer> versions, int threads, boolean slave, AutoFlush autoFlush) { + return new LuceneIndexModule(versions, threads, slave, autoFlush); } - public static LuceneIndexModule latestVersion(boolean slave) { - return new LuceneIndexModule(null, 0, slave); + public static LuceneIndexModule latestVersion(boolean slave, AutoFlush autoFlush) { + return new LuceneIndexModule(null, 0, slave, autoFlush); } static boolean isInMemoryTest(Config cfg) { return cfg.getBoolean("index", "lucene", "testInmemory", false); } - private LuceneIndexModule(Map<String, Integer> singleVersions, int threads, boolean slave) { + private LuceneIndexModule( + Map<String, Integer> singleVersions, int threads, boolean slave, AutoFlush autoFlush) { super(singleVersions, threads, slave); + this.autoFlush = autoFlush; + } + + @Override + protected void configure() { + super.configure(); + bind(AutoFlush.class).toInstance(autoFlush); } @Override diff --git a/java/com/google/gerrit/lucene/LuceneProjectIndex.java b/java/com/google/gerrit/lucene/LuceneProjectIndex.java index 2a418ca17b..7e7aecb923 100644 --- a/java/com/google/gerrit/lucene/LuceneProjectIndex.java +++ b/java/com/google/gerrit/lucene/LuceneProjectIndex.java @@ -32,6 +32,7 @@ import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.index.IndexUtils; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.gerrit.server.project.ProjectCache; import com.google.gerrit.server.project.ProjectState; import com.google.inject.Inject; @@ -84,7 +85,8 @@ public class LuceneProjectIndex extends AbstractLuceneIndex<Project.NameKey, Pro @GerritServerConfig Config cfg, SitePaths sitePaths, Provider<ProjectCache> projectCache, - @Assisted Schema<ProjectData> schema) + @Assisted Schema<ProjectData> schema, + AutoFlush autoFlush) throws IOException { super( schema, @@ -94,7 +96,8 @@ public class LuceneProjectIndex extends AbstractLuceneIndex<Project.NameKey, Pro ImmutableSet.of(), null, new GerritIndexWriterConfig(cfg, PROJECTS), - new SearcherFactory()); + new SearcherFactory(), + autoFlush); this.projectCache = projectCache; indexWriterConfig = new GerritIndexWriterConfig(cfg, PROJECTS); diff --git a/java/com/google/gerrit/pgm/Daemon.java b/java/com/google/gerrit/pgm/Daemon.java index f21a350000..5f675bcc95 100644 --- a/java/com/google/gerrit/pgm/Daemon.java +++ b/java/com/google/gerrit/pgm/Daemon.java @@ -87,6 +87,7 @@ import com.google.gerrit.server.group.PeriodicGroupIndexer; import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.OnlineUpgrader; import com.google.gerrit.server.index.VersionManager; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.gerrit.server.mail.SignedTokenEmailTokenVerifier; import com.google.gerrit.server.mail.receive.MailReceiver; import com.google.gerrit.server.mail.send.SmtpEmailSender; @@ -518,7 +519,7 @@ public class Daemon extends SiteProgram { return luceneModule; } if (indexType.isLucene()) { - return LuceneIndexModule.latestVersion(replica); + return LuceneIndexModule.latestVersion(replica, AutoFlush.ENABLED); } if (indexType.isElasticsearch()) { return ElasticIndexModule.latestVersion(replica); diff --git a/java/com/google/gerrit/pgm/Reindex.java b/java/com/google/gerrit/pgm/Reindex.java index 3935268fe9..0872340d23 100644 --- a/java/com/google/gerrit/pgm/Reindex.java +++ b/java/com/google/gerrit/pgm/Reindex.java @@ -33,6 +33,7 @@ import com.google.gerrit.server.change.ChangeResource; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.change.ChangeSchemaDefinitions; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.gerrit.server.plugins.PluginGuiceEnvironment; import com.google.gerrit.server.util.ReplicaUtil; import com.google.inject.Inject; @@ -52,8 +53,10 @@ import org.eclipse.jgit.util.io.NullOutputStream; import org.kohsuke.args4j.Option; public class Reindex extends SiteProgram { - @Option(name = "--threads", usage = "Number of threads to use for indexing") - private int threads = Runtime.getRuntime().availableProcessors(); + @Option( + name = "--threads", + usage = "Number of threads to use for indexing. Default is index.batchThreads from config.") + private int threads = 0; @Option( name = "--changes-schema-version", @@ -150,7 +153,9 @@ public class Reindex extends SiteProgram { Module indexModule; IndexType indexType = IndexModule.getIndexType(dbInjector); if (indexType.isLucene()) { - indexModule = LuceneIndexModule.singleVersionWithExplicitVersions(versions, threads, replica); + indexModule = + LuceneIndexModule.singleVersionWithExplicitVersions( + versions, threads, replica, AutoFlush.DISABLED); } else if (indexType.isElasticsearch()) { indexModule = ElasticIndexModule.singleVersionWithExplicitVersions(versions, threads, replica); diff --git a/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java b/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java index 12a44dc535..078e6488d8 100644 --- a/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java +++ b/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java @@ -19,6 +19,7 @@ import com.google.gerrit.lucene.LuceneGroupIndex; import com.google.gerrit.pgm.init.index.IndexModuleOnInit; import com.google.gerrit.server.index.account.AccountIndex; import com.google.gerrit.server.index.group.GroupIndex; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.inject.AbstractModule; import com.google.inject.assistedinject.FactoryModuleBuilder; @@ -36,5 +37,7 @@ public class LuceneIndexModuleOnInit extends AbstractModule { .build(GroupIndex.Factory.class)); install(new IndexModuleOnInit()); + + bind(AutoFlush.class).toInstance(AutoFlush.DISABLED); } } diff --git a/java/com/google/gerrit/server/git/meta/VersionedMetaData.java b/java/com/google/gerrit/server/git/meta/VersionedMetaData.java index feb038aa9e..ea594dd0c3 100644 --- a/java/com/google/gerrit/server/git/meta/VersionedMetaData.java +++ b/java/com/google/gerrit/server/git/meta/VersionedMetaData.java @@ -217,6 +217,27 @@ public abstract class VersionedMetaData { } /** + * Update this metadata branch, recording a new commit on its reference. This method mutates its + * receiver. + * + * @param update helper information to define the update that will occur. + * @param objInserter Shared object inserter. + * @param objReader Shared object reader. + * @param revWalk Shared rev walk. + * @return the commit that was created + * @throws IOException if there is a storage problem and the update cannot be executed as + * requested or if it failed because of a concurrent update to the same reference + */ + public RevCommit commit( + MetaDataUpdate update, ObjectInserter objInserter, ObjectReader objReader, RevWalk revWalk) + throws IOException { + try (BatchMetaDataUpdate batch = openUpdate(update, objInserter, objReader, revWalk)) { + batch.write(update.getCommitBuilder()); + return batch.commit(); + } + } + + /** * Creates a new commit and a new ref based on this commit. This method mutates its receiver. * * @param update helper information to define the update that will occur. @@ -263,11 +284,39 @@ public abstract class VersionedMetaData { * @throws IOException if the update failed. */ public BatchMetaDataUpdate openUpdate(MetaDataUpdate update) throws IOException { + return openUpdate(update, null, null, null); + } + + /** + * Open a batch of updates to the same metadata ref. + * + * <p>This allows making multiple commits to a single metadata ref, at the end of which is a + * single ref update. For batching together updates to multiple refs (each consisting of one or + * more commits against their respective refs), create the {@link MetaDataUpdate} with a {@link + * BatchRefUpdate}. + * + * <p>A ref update produced by this {@link BatchMetaDataUpdate} is only committed if there is no + * associated {@link BatchRefUpdate}. As a result, the configured ref updated event is not fired + * if there is an associated batch. + * + * <p>If object inserter, reader and revwalk are provided, then the updates are not flushed, + * allowing callers the flexibility to flush only once after several updates. + * + * @param update helper info about the update. + * @param objInserter Shared object inserter. + * @param objReader Shared object reader. + * @param revWalk Shared rev walk. + * @throws IOException if the update failed. + */ + public BatchMetaDataUpdate openUpdate( + MetaDataUpdate update, ObjectInserter objInserter, ObjectReader objReader, RevWalk revWalk) + throws IOException { final Repository db = update.getRepository(); - inserter = db.newObjectInserter(); - reader = inserter.newReader(); - final RevWalk rw = new RevWalk(reader); + inserter = objInserter == null ? db.newObjectInserter() : objInserter; + reader = objReader == null ? inserter.newReader() : objReader; + final RevWalk rw = revWalk == null ? new RevWalk(reader) : revWalk; + final RevTree tree = revision != null ? rw.parseTree(revision) : null; newTree = readTree(tree); return new BatchMetaDataUpdate() { @@ -379,13 +428,16 @@ public abstract class VersionedMetaData { public void close() { newTree = null; - rw.close(); - if (inserter != null) { + if (revWalk == null) { + rw.close(); + } + + if (objInserter == null && inserter != null) { inserter.close(); inserter = null; } - if (reader != null) { + if (objReader == null && reader != null) { reader.close(); reader = null; } @@ -396,7 +448,9 @@ public abstract class VersionedMetaData { BatchRefUpdate bru = update.getBatch(); if (bru != null) { bru.addCommand(new ReceiveCommand(oldId.toObjectId(), newId.toObjectId(), refName)); - inserter.flush(); + if (objInserter == null) { + inserter.flush(); + } revision = rw.parseCommit(newId); return revision; } diff --git a/java/com/google/gerrit/server/index/IndexModule.java b/java/com/google/gerrit/server/index/IndexModule.java index 6db00f5a4b..17665c0903 100644 --- a/java/com/google/gerrit/server/index/IndexModule.java +++ b/java/com/google/gerrit/server/index/IndexModule.java @@ -208,13 +208,14 @@ public class IndexModule extends LifecycleModule { return interactiveExecutor; } int threads = this.threads; - if (threads < 0) { - return MoreExecutors.newDirectExecutorService(); - } else if (threads == 0) { + if (threads == 0) { threads = config.getInt( "index", null, "threads", Runtime.getRuntime().availableProcessors() / 2 + 1); } + if (threads < 0) { + return MoreExecutors.newDirectExecutorService(); + } return MoreExecutors.listeningDecorator( workQueue.createQueue(threads, "Index-Interactive", true)); } @@ -227,11 +228,13 @@ public class IndexModule extends LifecycleModule { if (batchExecutor != null) { return batchExecutor; } - int threads = config.getInt("index", null, "batchThreads", 0); + int threads = this.threads; + if (threads == 0) { + threads = + config.getInt("index", null, "batchThreads", Runtime.getRuntime().availableProcessors()); + } if (threads < 0) { return MoreExecutors.newDirectExecutorService(); - } else if (threads == 0) { - threads = Runtime.getRuntime().availableProcessors(); } return MoreExecutors.listeningDecorator(workQueue.createQueue(threads, "Index-Batch", true)); } diff --git a/java/com/google/gerrit/server/index/change/AllChangesIndexer.java b/java/com/google/gerrit/server/index/change/AllChangesIndexer.java index 832dca63ff..01efe1a2e6 100644 --- a/java/com/google/gerrit/server/index/change/AllChangesIndexer.java +++ b/java/com/google/gerrit/server/index/change/AllChangesIndexer.java @@ -134,6 +134,10 @@ public class AllChangesIndexer extends SiteIndexer<Change.Id, ChangeData, Change // splitting of repos into smaller parts reduced indexing time from 1.5 hours to 55 minutes // in 2020. int size = estimateSize(repo); + if (size == 0) { + pm.update(1); + continue; + } changeCount += size; int slices = 1 + size / PROJECT_SLICE_MAX_REFS; if (slices > 1) { diff --git a/java/com/google/gerrit/server/index/options/AutoFlush.java b/java/com/google/gerrit/server/index/options/AutoFlush.java new file mode 100644 index 0000000000..7b82edb430 --- /dev/null +++ b/java/com/google/gerrit/server/index/options/AutoFlush.java @@ -0,0 +1,20 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.options; + +public enum AutoFlush { + ENABLED, + DISABLED +} diff --git a/java/com/google/gerrit/server/mail/send/OutgoingEmail.java b/java/com/google/gerrit/server/mail/send/OutgoingEmail.java index bef5317ba2..a23c978347 100644 --- a/java/com/google/gerrit/server/mail/send/OutgoingEmail.java +++ b/java/com/google/gerrit/server/mail/send/OutgoingEmail.java @@ -142,7 +142,8 @@ public abstract class OutgoingEmail { // drop them from the recipient lists. // logger.atFine().log( - "Not CCing email sender %s because the email strategy of this user is not %s but %s", + "Not CCing email sender %s because the email strategy of this user is not %s but" + + " %s", fromUser.get().account().id(), CC_ON_OWN_COMMENTS, senderPrefs != null ? senderPrefs.getEmailStrategy() : null); @@ -422,9 +423,13 @@ public abstract class OutgoingEmail { * username. If no username is set, this function returns null. * * @param accountId user to fetch. - * @return name/email of account, username, or null if unset. + * @return name/email of account, username, or null if unset or the accountId is null. */ - protected String getUserNameEmailFor(Account.Id accountId) { + protected String getUserNameEmailFor(@Nullable Account.Id accountId) { + if (accountId == null) { + return null; + } + Optional<AccountState> accountState = args.accountCache.get(accountId); if (!accountState.isPresent()) { return null; diff --git a/java/com/google/gerrit/server/restapi/project/GetAccess.java b/java/com/google/gerrit/server/restapi/project/GetAccess.java index b572db31d6..a79439c1b4 100644 --- a/java/com/google/gerrit/server/restapi/project/GetAccess.java +++ b/java/com/google/gerrit/server/restapi/project/GetAccess.java @@ -331,7 +331,7 @@ public class GetAccess implements RestReadView<ProjectResource> { } AccountGroup.UUID group = r.getGroup().getUUID(); if (group != null) { - pInfo.rules.put(group.get(), info); + pInfo.rules.putIfAbsent(group.get(), info); // First entry for the group wins loadGroup(groups, group); } } diff --git a/javatests/com/google/gerrit/acceptance/rest/project/AccessIT.java b/javatests/com/google/gerrit/acceptance/rest/project/AccessIT.java index 33d0d29086..caf9b907e7 100644 --- a/javatests/com/google/gerrit/acceptance/rest/project/AccessIT.java +++ b/javatests/com/google/gerrit/acceptance/rest/project/AccessIT.java @@ -23,6 +23,8 @@ import static com.google.gerrit.server.schema.AclUtil.grant; import static com.google.gerrit.testing.GerritJUnit.assertThrows; import static com.google.gerrit.truth.ConfigSubject.assertThat; import static com.google.gerrit.truth.MapSubject.assertThatMap; +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; import com.google.gerrit.acceptance.AbstractDaemonTest; import com.google.gerrit.acceptance.ExtensionRegistry; @@ -62,6 +64,7 @@ import com.google.gerrit.server.project.ProjectConfig; import com.google.gerrit.server.schema.GrantRevertPermission; import com.google.inject.Inject; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; import org.eclipse.jgit.junit.TestRepository; @@ -941,6 +944,90 @@ public class AccessIT extends AbstractDaemonTest { return gApi.projects().name(newProjectName.get()); } + @Test + public void grantAllowAndDenyForSameGroup() throws Exception { + GroupReference registeredUsers = systemGroupBackend.getGroup(REGISTERED_USERS); + String access = "access"; + List<String> allowThenDeny = + asList(registeredUsers.toConfigValue(), "deny " + registeredUsers.toConfigValue()); + // Clone repository to forcefully add permission + TestRepository<InMemoryRepository> allProjectsRepo = cloneProject(allProjects, admin); + + // Fetch permission ref + GitUtil.fetch(allProjectsRepo, "refs/meta/config:cfg"); + allProjectsRepo.reset("cfg"); + + // Load current permissions + String config = + gApi.projects() + .name(allProjects.get()) + .branch(RefNames.REFS_CONFIG) + .file(ProjectConfig.PROJECT_CONFIG) + .asString(); + + // Append and push allowThenDeny permissions + Config cfg = new Config(); + cfg.fromText(config); + cfg.setStringList(access, AccessSection.HEADS, Permission.READ, allowThenDeny); + config = cfg.toText(); + PushOneCommit push = + pushFactory.create( + admin.newIdent(), allProjectsRepo, "Subject", ProjectConfig.PROJECT_CONFIG, config); + push.to(RefNames.REFS_CONFIG).assertOkStatus(); + + ProjectAccessInfo pai = gApi.projects().name(allProjects.get()).access(); + Map<String, AccessSectionInfo> local = pai.local; + AccessSectionInfo heads = local.get(AccessSection.HEADS); + Map<String, PermissionInfo> permissions = heads.permissions; + PermissionInfo read = permissions.get(Permission.READ); + Map<String, PermissionRuleInfo> rules = read.rules; + assertEquals( + rules.get(registeredUsers.getUUID().get()), + new PermissionRuleInfo(PermissionRuleInfo.Action.ALLOW, false)); + } + + @Test + public void grantDenyAndAllowForSameGroup() throws Exception { + GroupReference registeredUsers = systemGroupBackend.getGroup(REGISTERED_USERS); + String access = "access"; + List<String> denyThenAllow = + asList("deny " + registeredUsers.toConfigValue(), registeredUsers.toConfigValue()); + // Clone repository to forcefully add permission + TestRepository<InMemoryRepository> allProjectsRepo = cloneProject(allProjects, admin); + + // Fetch permission ref + GitUtil.fetch(allProjectsRepo, "refs/meta/config:cfg"); + allProjectsRepo.reset("cfg"); + + // Load current permissions + String config = + gApi.projects() + .name(allProjects.get()) + .branch(RefNames.REFS_CONFIG) + .file(ProjectConfig.PROJECT_CONFIG) + .asString(); + + // Append and push denyThenAllow permissions + Config cfg = new Config(); + cfg.fromText(config); + cfg.setStringList(access, AccessSection.HEADS, Permission.READ, denyThenAllow); + config = cfg.toText(); + PushOneCommit push = + pushFactory.create( + admin.newIdent(), allProjectsRepo, "Subject", ProjectConfig.PROJECT_CONFIG, config); + push.to(RefNames.REFS_CONFIG).assertOkStatus(); + + ProjectAccessInfo pai = gApi.projects().name(allProjects.get()).access(); + Map<String, AccessSectionInfo> local = pai.local; + AccessSectionInfo heads = local.get(AccessSection.HEADS); + Map<String, PermissionInfo> permissions = heads.permissions; + PermissionInfo read = permissions.get(Permission.READ); + Map<String, PermissionRuleInfo> rules = read.rules; + assertEquals( + rules.get(registeredUsers.getUUID().get()), + new PermissionRuleInfo(PermissionRuleInfo.Action.DENY, false)); + } + private ProjectAccessInput newProjectAccessInput() { ProjectAccessInput p = new ProjectAccessInput(); p.add = new HashMap<>(); diff --git a/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor.ts b/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor.ts index 1ab32ead0e..ac82547faa 100644 --- a/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor.ts +++ b/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor.ts @@ -25,7 +25,6 @@ import {LegacyElementMixin} from '@polymer/polymer/lib/legacy/legacy-element-mix import {PolymerElement} from '@polymer/polymer/polymer-element'; import {htmlTemplate} from './gr-plugin-config-array-editor_html'; import {property, customElement} from '@polymer/decorators'; -import {PolymerDeepPropertyChange} from '@polymer/polymer/interfaces'; import { PluginConfigOptionsChangedEventDetail, ArrayPluginOption, @@ -59,19 +58,8 @@ class GrPluginConfigArrayEditor extends GestureEventListeners( @property({type: Object}) pluginOption!: ArrayPluginOption; - @property({type: Boolean, computed: '_computeDisabled(pluginOption.*)'}) - disabled?: boolean; - - _computeDisabled( - record: PolymerDeepPropertyChange<ArrayPluginOption, ArrayPluginOption> - ) { - return !( - record && - record.base && - record.base.info && - record.base.info.editable - ); - } + @property({type: Boolean, reflectToAttribute: true}) + disabled = false; _handleAddTap(e: MouseEvent) { e.preventDefault(); diff --git a/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_html.ts b/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_html.ts index c96b86c56f..7709198604 100644 --- a/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_html.ts +++ b/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_html.ts @@ -85,6 +85,7 @@ export const htmlTemplate = html` id="input" on-keydown="_handleInputKeydown" bind-value="{{_newValue}}" + disabled$="[[disabled]]" /> </iron-input> <gr-button diff --git a/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_test.js b/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_test.js index dfc191f228..326ff44e54 100644 --- a/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_test.js +++ b/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_test.js @@ -42,16 +42,6 @@ suite('gr-plugin-config-array-editor tests', () => { assert.equal(element._computeShowInputRow(false), ''); }); - test('_computeDisabled', () => { - assert.isTrue(element._computeDisabled({})); - assert.isTrue(element._computeDisabled({base: {}})); - assert.isTrue(element._computeDisabled({base: {info: {}}})); - assert.isTrue( - element._computeDisabled({base: {info: {editable: false}}})); - assert.isFalse( - element._computeDisabled({base: {info: {editable: true}}})); - }); - suite('adding', () => { setup(() => { dispatchStub = sinon.stub(element, '_dispatchChanged'); @@ -60,11 +50,13 @@ suite('gr-plugin-config-array-editor tests', () => { test('with enter', () => { element._newValue = ''; MockInteractions.pressAndReleaseKeyOn(element.$.input, 13); // Enter + assert.isFalse(element.$.input.hasAttribute('disabled')); flush(); assert.isFalse(dispatchStub.called); element._newValue = 'test'; MockInteractions.pressAndReleaseKeyOn(element.$.input, 13); // Enter + assert.isFalse(element.$.input.hasAttribute('disabled')); flush(); assert.isTrue(dispatchStub.called); @@ -91,6 +83,7 @@ suite('gr-plugin-config-array-editor tests', () => { test('deleting', () => { dispatchStub = sinon.stub(element, '_dispatchChanged'); element.pluginOption = {info: {values: ['test', 'test2']}}; + element.disabled = true; flush(); const rows = getAll('.existingItems .row'); @@ -101,7 +94,7 @@ suite('gr-plugin-config-array-editor tests', () => { flush(); assert.isFalse(dispatchStub.called); - element.pluginOption.info.editable = true; + element.disabled = false; element.notifyPath('pluginOption.info.editable'); flush(); diff --git a/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config.ts b/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config.ts index 880b397b37..e7eb9dc82d 100644 --- a/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config.ts +++ b/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config.ts @@ -84,6 +84,9 @@ class GrRepoPluginConfig extends GestureEventListeners( }) _pluginConfigOptions!: PluginOption[]; // _computePluginConfigOptions never returns null + @property({type: Boolean, reflectToAttribute: true}) + disabled = false; + _computePluginConfigOptions( dataRecord: PolymerDeepPropertyChange<PluginData, PluginData> ): PluginOption[] { @@ -117,8 +120,8 @@ class GrRepoPluginConfig extends GestureEventListeners( ); } - _computeDisabled(editable: boolean) { - return !editable; + _computeDisabled(disabled: boolean, editable: boolean) { + return disabled || !editable; } _computeChecked(value = 'false') { diff --git a/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_html.ts b/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_html.ts index 3045108558..208e042e3e 100644 --- a/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_html.ts +++ b/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_html.ts @@ -55,6 +55,7 @@ export const htmlTemplate = html` <gr-plugin-config-array-editor on-plugin-config-option-changed="_handleArrayChange" plugin-option="[[option]]" + disabled$="[[_computeDisabled(disabled, option.info.editable)]]" ></gr-plugin-config-array-editor> </template> <template is="dom-if" if="[[_isBoolean(option.info.type)]]"> @@ -62,7 +63,7 @@ export const htmlTemplate = html` checked="[[_computeChecked(option.info.value)]]" on-change="_handleBooleanChange" data-option-key$="[[option._key]]" - disabled$="[[_computeDisabled(option.info.editable)]]" + disabled$="[[_computeDisabled(disabled, option.info.editable)]]" on-tap="_onTapPluginBoolean" ></paper-toggle-button> </template> @@ -73,7 +74,7 @@ export const htmlTemplate = html` > <select data-option-key$="[[option._key]]" - disabled$="[[_computeDisabled(option.info.editable)]]" + disabled$="[[_computeDisabled(disabled, option.info.editable)]]" > <template is="dom-repeat" @@ -90,14 +91,14 @@ export const htmlTemplate = html` bind-value="[[option.info.value]]" on-input="_handleStringChange" data-option-key$="[[option._key]]" - disabled$="[[_computeDisabled(option.info.editable)]]" + disabled$="[[_computeDisabled(disabled, option.info.editable)]]" > <input is="iron-input" value="[[option.info.value]]" on-input="_handleStringChange" data-option-key$="[[option._key]]" - disabled$="[[_computeDisabled(option.info.editable)]]" + disabled$="[[_computeDisabled(disabled, option.info.editable)]]" /> </iron-input> </template> diff --git a/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_test.js b/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_test.js index 28ef2f8dd1..a5c7dfef2b 100644 --- a/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_test.js +++ b/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_test.js @@ -39,10 +39,11 @@ suite('gr-repo-plugin-config tests', () => { }); test('_computeDisabled', () => { - assert.isFalse(element._computeDisabled(true)); - assert.isTrue(element._computeDisabled(undefined)); - assert.isTrue(element._computeDisabled(null)); - assert.isTrue(element._computeDisabled(false)); + assert.isFalse(element._computeDisabled(false, true)); + assert.isTrue(element._computeDisabled(false, undefined)); + assert.isTrue(element._computeDisabled(false, null)); + assert.isTrue(element._computeDisabled(false, false)); + assert.isTrue(element._computeDisabled(true, true)); }); test('_handleChange', () => { diff --git a/polygerrit-ui/app/elements/admin/gr-repo/gr-repo_html.ts b/polygerrit-ui/app/elements/admin/gr-repo/gr-repo_html.ts index 2e86758e27..3d96f9895e 100644 --- a/polygerrit-ui/app/elements/admin/gr-repo/gr-repo_html.ts +++ b/polygerrit-ui/app/elements/admin/gr-repo/gr-repo_html.ts @@ -414,6 +414,7 @@ export const htmlTemplate = html` <template is="dom-repeat" items="[[_pluginData]]" as="data"> <gr-repo-plugin-config plugin-data="[[data]]" + disabled$="[[_readOnly]]" ></gr-repo-plugin-config> </template> </div> diff --git a/resources/com/google/gerrit/httpd/auth/container/LoginRedirect.html b/resources/com/google/gerrit/httpd/auth/container/LoginRedirect.html index 0567468e8c..54c366148d 100644 --- a/resources/com/google/gerrit/httpd/auth/container/LoginRedirect.html +++ b/resources/com/google/gerrit/httpd/auth/container/LoginRedirect.html @@ -4,6 +4,12 @@ <title>Gerrit Code Review</title> <script type="text/javascript"> var href = window.location.href; + var query = ""; + var q = href.indexOf('?'); + if (q >= 0) { + query = href.substring(q); + href = href.substring(0,q); + } var p = href.indexOf('#'); var token; if (p >= 0) { @@ -12,7 +18,7 @@ } else { token = ''; } - window.location.replace(href + 'login/' + token); + window.location.replace(href + 'login/' + token + query); </script> </head> <body> |