diff options
author | Luca Milanesio <luca.milanesio@gmail.com> | 2021-11-22 09:29:11 +0000 |
---|---|---|
committer | Luca Milanesio <luca.milanesio@gmail.com> | 2021-11-22 10:48:16 +0000 |
commit | f53bbf3464526928838b53bd959320c2dad61b42 (patch) | |
tree | 0fb87d4460820e2d958f741e1c97d89eeab869e5 | |
parent | 5959ebfe59fa611585b434f46e249699a105c7c4 (diff) | |
parent | 57341128f1d55ee9fb318e8a96311f6dcb52cf9e (diff) |
Merge branch 'stable-3.2' into stable-3.3
* stable-3.2:
Set version to 3.2.15-SNAPSHOT
Set version to 3.2.14
Set version to 2.16.29-SNAPSHOT
Set version to 2.16.28
Use JGit 5.1.15.202012011955-r javadoc to resume release build
Avoid lucene index deletes during offline reindexing
Remove bazel workaround
Fix group suggestions
Display cache stats after reindex operation
Fix AllProjectsIndexer to avoid duplicate reindex work
Rename ProjectCache.evict() to ProjectCache.evictAndReindex()
Change-Id: I6f672d580119bf1da4fb84a8ed16199e9d103899
58 files changed, 534 insertions, 277 deletions
@@ -11,10 +11,6 @@ build --java_toolchain=//tools:error_prone_warnings_toolchain_java11 # this flag here once flipped in Bazel again. build --incompatible_strict_action_env -# Workaround Bazel worker crash (remove after upgrading to 4.1.0) -# https://github.com/bazelbuild/bazel/issues/13333 -build --experimental_worker_multiplex=false - test --build_tests_only test --test_output=errors test --java_toolchain=//tools:error_prone_warnings_toolchain_java11 diff --git a/java/com/google/gerrit/acceptance/AbstractDaemonTest.java b/java/com/google/gerrit/acceptance/AbstractDaemonTest.java index 4ab5d51c35..f44a78d0a6 100644 --- a/java/com/google/gerrit/acceptance/AbstractDaemonTest.java +++ b/java/com/google/gerrit/acceptance/AbstractDaemonTest.java @@ -1028,7 +1028,7 @@ public abstract class AbstractDaemonTest { ProjectConfig config = projectConfigFactory.read(md); config.updateProject(p -> p.setBooleanConfig(BooleanProjectConfig.USE_SIGNED_OFF_BY, value)); config.commit(md); - projectCache.evict(config.getProject()); + projectCache.evictAndReindex(config.getProject()); } } @@ -1037,7 +1037,7 @@ public abstract class AbstractDaemonTest { ProjectConfig config = projectConfigFactory.read(md); config.updateProject(p -> p.setBooleanConfig(BooleanProjectConfig.REQUIRE_CHANGE_ID, value)); config.commit(md); - projectCache.evict(config.getProject()); + projectCache.evictAndReindex(config.getProject()); } } @@ -1532,7 +1532,7 @@ public abstract class AbstractDaemonTest { projectConfig.commit(metaDataUpdate); metaDataUpdate.close(); metaDataUpdate = null; - projectCache.evict(projectConfig.getProject()); + projectCache.evictAndReindex(projectConfig.getProject()); } @Override diff --git a/java/com/google/gerrit/acceptance/DisabledAccountIndex.java b/java/com/google/gerrit/acceptance/DisabledAccountIndex.java index 271d15c176..7bd0c73013 100644 --- a/java/com/google/gerrit/acceptance/DisabledAccountIndex.java +++ b/java/com/google/gerrit/acceptance/DisabledAccountIndex.java @@ -45,6 +45,11 @@ public class DisabledAccountIndex implements AccountIndex { } @Override + public void insert(AccountState obj) { + throw new UnsupportedOperationException("AccountIndex is disabled"); + } + + @Override public void replace(AccountState obj) { throw new UnsupportedOperationException("AccountIndex is disabled"); } diff --git a/java/com/google/gerrit/acceptance/DisabledChangeIndex.java b/java/com/google/gerrit/acceptance/DisabledChangeIndex.java index 34f72f5ccf..7671ad4068 100644 --- a/java/com/google/gerrit/acceptance/DisabledChangeIndex.java +++ b/java/com/google/gerrit/acceptance/DisabledChangeIndex.java @@ -52,6 +52,11 @@ public class DisabledChangeIndex implements ChangeIndex { } @Override + public void insert(ChangeData obj) { + throw new UnsupportedOperationException("ChangeIndex is disabled"); + } + + @Override public void replace(ChangeData obj) { throw new UnsupportedOperationException("ChangeIndex is disabled"); } diff --git a/java/com/google/gerrit/acceptance/DisabledProjectIndex.java b/java/com/google/gerrit/acceptance/DisabledProjectIndex.java index ed119ff3bc..2e3dd906b6 100644 --- a/java/com/google/gerrit/acceptance/DisabledProjectIndex.java +++ b/java/com/google/gerrit/acceptance/DisabledProjectIndex.java @@ -50,6 +50,11 @@ public class DisabledProjectIndex implements ProjectIndex { } @Override + public void insert(ProjectData obj) { + throw new UnsupportedOperationException("ProjectIndex is disabled"); + } + + @Override public void replace(ProjectData obj) { throw new UnsupportedOperationException("ProjectIndex is disabled"); } diff --git a/java/com/google/gerrit/acceptance/ProjectResetter.java b/java/com/google/gerrit/acceptance/ProjectResetter.java index d8853033d2..46f7496e82 100644 --- a/java/com/google/gerrit/acceptance/ProjectResetter.java +++ b/java/com/google/gerrit/acceptance/ProjectResetter.java @@ -307,7 +307,7 @@ public class ProjectResetter implements AutoCloseable { Sets.union( projectsWithConfigChanges(restoredRefsByProject), projectsWithConfigChanges(deletedRefsByProject))) { - projectCache.evict(project); + projectCache.evictAndReindex(project); } } diff --git a/java/com/google/gerrit/acceptance/ReadOnlyChangeIndex.java b/java/com/google/gerrit/acceptance/ReadOnlyChangeIndex.java index e943519d19..f7a0669739 100644 --- a/java/com/google/gerrit/acceptance/ReadOnlyChangeIndex.java +++ b/java/com/google/gerrit/acceptance/ReadOnlyChangeIndex.java @@ -45,6 +45,11 @@ class ReadOnlyChangeIndex implements ChangeIndex { } @Override + public void insert(ChangeData obj) { + // do nothing + } + + @Override public void replace(ChangeData obj) { // do nothing } diff --git a/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java b/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java index e7354ab473..394f0f8280 100644 --- a/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java +++ b/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java @@ -148,7 +148,7 @@ public class ProjectOperationsImpl implements ProjectOperations { setExclusiveGroupPermissions(projectConfig, projectUpdate.exclusiveGroupPermissions()); projectConfig.commit(metaDataUpdate); } - projectCache.evict(nameKey); + projectCache.evictAndReindex(nameKey); } private void removePermissions( @@ -293,7 +293,7 @@ public class ProjectOperationsImpl implements ProjectOperations { setConfig(projectConfig); try { - projectCache.evict(nameKey); + projectCache.evictAndReindex(nameKey); } catch (Exception e) { // Evicting the project from the cache, also triggers a reindex of the project. // The reindex step fails if the project config is invalid. That's fine, since it was our @@ -311,7 +311,7 @@ public class ProjectOperationsImpl implements ProjectOperations { testProjectInvalidation.projectConfigUpdater().forEach(c -> c.accept(projectConfig)); setConfig(projectConfig); try { - projectCache.evict(nameKey); + projectCache.evictAndReindex(nameKey); } catch (Exception e) { // Evicting the project from the cache, also triggers a reindex of the project. // The reindex step fails if the project config is invalid. That's fine, since it was our diff --git a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java index e56f470300..d53490b8f6 100644 --- a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java +++ b/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java @@ -175,6 +175,12 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> { } @Override + public void insert(V obj) { + // TODO: Implement real insert() if it helps performance + replace(obj); + } + + @Override public void deleteAll() { // Delete the index, if it exists. String endpoint = indexName + client.adapter().indicesExistParams(); diff --git a/java/com/google/gerrit/index/Index.java b/java/com/google/gerrit/index/Index.java index 44f8b4236e..e662bc8477 100644 --- a/java/com/google/gerrit/index/Index.java +++ b/java/com/google/gerrit/index/Index.java @@ -40,6 +40,16 @@ public interface Index<K, V> { void close(); /** + * Insert a document into the index. + * + * <p>Results may not be immediately visible to searchers, but should be visible within a + * reasonable amount of time. + * + * @param obj document object + */ + void insert(V obj); + + /** * Update a document in the index. * * <p>Semantically equivalent to deleting the document and reinserting it with new field values. A diff --git a/java/com/google/gerrit/lucene/ChangeSubIndex.java b/java/com/google/gerrit/lucene/ChangeSubIndex.java index 5c24b9a009..360331a7bd 100644 --- a/java/com/google/gerrit/lucene/ChangeSubIndex.java +++ b/java/com/google/gerrit/lucene/ChangeSubIndex.java @@ -89,6 +89,11 @@ public class ChangeSubIndex extends AbstractLuceneIndex<Change.Id, ChangeData> } @Override + public void insert(ChangeData obj) { + throw new UnsupportedOperationException("don't use ChangeSubIndex directly"); + } + + @Override public void replace(ChangeData obj) { throw new UnsupportedOperationException("don't use ChangeSubIndex directly"); } diff --git a/java/com/google/gerrit/lucene/LuceneAccountIndex.java b/java/com/google/gerrit/lucene/LuceneAccountIndex.java index 87b7ccef6b..c4a524060d 100644 --- a/java/com/google/gerrit/lucene/LuceneAccountIndex.java +++ b/java/com/google/gerrit/lucene/LuceneAccountIndex.java @@ -144,6 +144,15 @@ public class LuceneAccountIndex extends AbstractLuceneIndex<Account.Id, AccountS } @Override + public void insert(AccountState as) { + try { + insert(toDocument(as)).get(); + } catch (ExecutionException | InterruptedException e) { + throw new StorageException(e); + } + } + + @Override public void delete(Account.Id key) { try { delete(idTerm(getSchema().useLegacyNumericFields(), key)).get(); diff --git a/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/java/com/google/gerrit/lucene/LuceneChangeIndex.java index bb97936bad..78f4624a91 100644 --- a/java/com/google/gerrit/lucene/LuceneChangeIndex.java +++ b/java/com/google/gerrit/lucene/LuceneChangeIndex.java @@ -287,6 +287,23 @@ public class LuceneChangeIndex implements ChangeIndex { } @Override + public void insert(ChangeData cd) { + Term id = LuceneChangeIndex.idTerm(idTerm, idField, cd); + // toDocument is essentially static and doesn't depend on the specific + // sub-index, so just pick one. + Document doc = openIndex.toDocument(cd); + try { + if (cd.change().isNew()) { + openIndex.insert(doc).get(); + } else { + closedIndex.insert(doc).get(); + } + } catch (ExecutionException | InterruptedException e) { + throw new StorageException(e); + } + } + + @Override public void delete(Change.Id changeId) { Term id = LuceneChangeIndex.idTerm(idTerm, idField, changeId); try { diff --git a/java/com/google/gerrit/lucene/LuceneGroupIndex.java b/java/com/google/gerrit/lucene/LuceneGroupIndex.java index c1b44c7829..9f8b62e741 100644 --- a/java/com/google/gerrit/lucene/LuceneGroupIndex.java +++ b/java/com/google/gerrit/lucene/LuceneGroupIndex.java @@ -125,6 +125,15 @@ public class LuceneGroupIndex extends AbstractLuceneIndex<AccountGroup.UUID, Int } @Override + public void insert(InternalGroup group) { + try { + insert(toDocument(group)).get(); + } catch (ExecutionException | InterruptedException e) { + throw new StorageException(e); + } + } + + @Override public void delete(AccountGroup.UUID key) { try { delete(idTerm(key)).get(); diff --git a/java/com/google/gerrit/lucene/LuceneProjectIndex.java b/java/com/google/gerrit/lucene/LuceneProjectIndex.java index 7e7aecb923..11707be379 100644 --- a/java/com/google/gerrit/lucene/LuceneProjectIndex.java +++ b/java/com/google/gerrit/lucene/LuceneProjectIndex.java @@ -125,6 +125,15 @@ public class LuceneProjectIndex extends AbstractLuceneIndex<Project.NameKey, Pro } @Override + public void insert(ProjectData projectState) { + try { + insert(toDocument(projectState)).get(); + } catch (ExecutionException | InterruptedException e) { + throw new StorageException(e); + } + } + + @Override public void delete(Project.NameKey nameKey) { try { delete(idTerm(nameKey)).get(); diff --git a/java/com/google/gerrit/pgm/Reindex.java b/java/com/google/gerrit/pgm/Reindex.java index 0872340d23..9f2e7a7105 100644 --- a/java/com/google/gerrit/pgm/Reindex.java +++ b/java/com/google/gerrit/pgm/Reindex.java @@ -17,10 +17,12 @@ package com.google.gerrit.pgm; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toSet; +import com.google.common.cache.Cache; import com.google.common.collect.Sets; import com.google.gerrit.common.Die; import com.google.gerrit.elasticsearch.ElasticIndexModule; import com.google.gerrit.extensions.config.FactoryModule; +import com.google.gerrit.extensions.registration.DynamicMap; import com.google.gerrit.index.Index; import com.google.gerrit.index.IndexDefinition; import com.google.gerrit.index.IndexType; @@ -29,17 +31,24 @@ import com.google.gerrit.lifecycle.LifecycleManager; import com.google.gerrit.lucene.LuceneIndexModule; import com.google.gerrit.pgm.util.BatchProgramModule; import com.google.gerrit.pgm.util.SiteProgram; +import com.google.gerrit.server.cache.CacheDisplay; +import com.google.gerrit.server.cache.CacheInfo; import com.google.gerrit.server.change.ChangeResource; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.change.ChangeSchemaDefinitions; import com.google.gerrit.server.index.options.AutoFlush; +import com.google.gerrit.server.index.options.IsFirstInsertForEntry; import com.google.gerrit.server.plugins.PluginGuiceEnvironment; import com.google.gerrit.server.util.ReplicaUtil; +import com.google.inject.AbstractModule; import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; +import com.google.inject.multibindings.OptionalBinder; +import java.io.StringWriter; +import java.io.Writer; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -48,6 +57,8 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.util.io.NullOutputStream; import org.kohsuke.args4j.Option; @@ -78,6 +89,7 @@ public class Reindex extends SiteProgram { private Config globalConfig; @Inject private Collection<IndexDefinition<?, ?, ?>> indexDefs; + @Inject private DynamicMap<Cache<?, ?>> cacheMap; @Override public int run() throws Exception { @@ -163,6 +175,16 @@ public class Reindex extends SiteProgram { throw new IllegalStateException("unsupported index.type = " + indexType); } modules.add(indexModule); + modules.add( + new AbstractModule() { + @Override + protected void configure() { + super.configure(); + OptionalBinder.newOptionalBinder(binder(), IsFirstInsertForEntry.class) + .setBinding() + .toInstance(IsFirstInsertForEntry.YES); + } + }); modules.add(new BatchProgramModule(dbInjector)); modules.add( new FactoryModule() { @@ -210,6 +232,20 @@ public class Reindex extends SiteProgram { System.out.format( "Index %s in version %d is %sready\n", def.getName(), index.getSchema().getVersion(), result.success() ? "" : "NOT "); + + try (Writer sw = new StringWriter()) { + sw.write(String.format("Cache Statistics at the end of reindexing %s\n", def.getName())); + new CacheDisplay( + sw, + StreamSupport.stream(cacheMap.spliterator(), false) + .map(e -> new CacheInfo(e.getExportName(), e.get())) + .collect(Collectors.toList())) + .displayCaches(); + System.out.print(sw.toString()); + } catch (Exception e) { + System.out.format("Error displaying the cache statistics\n" + e.getMessage()); + } + return result.success(); } } diff --git a/java/com/google/gerrit/pgm/util/BatchProgramModule.java b/java/com/google/gerrit/pgm/util/BatchProgramModule.java index 0d378555b7..af01343883 100644 --- a/java/com/google/gerrit/pgm/util/BatchProgramModule.java +++ b/java/com/google/gerrit/pgm/util/BatchProgramModule.java @@ -127,7 +127,7 @@ public class BatchProgramModule extends FactoryModule { // We're just running through each change // once, so don't worry about cache removal. bind(new TypeLiteral<DynamicSet<CacheRemovalListener>>() {}).toInstance(DynamicSet.emptySet()); - bind(new TypeLiteral<DynamicMap<Cache<?, ?>>>() {}).toInstance(DynamicMap.emptyMap()); + DynamicMap.mapOf(binder(), new TypeLiteral<Cache<?, ?>>() {}); bind(new TypeLiteral<List<CommentLinkInfo>>() {}) .toProvider(CommentLinkProvider.class) .in(SINGLETON); diff --git a/java/com/google/gerrit/server/CreateGroupPermissionSyncer.java b/java/com/google/gerrit/server/CreateGroupPermissionSyncer.java index 6c76de7c62..60f3d4bbf0 100644 --- a/java/com/google/gerrit/server/CreateGroupPermissionSyncer.java +++ b/java/com/google/gerrit/server/CreateGroupPermissionSyncer.java @@ -121,7 +121,7 @@ public class CreateGroupPermissionSyncer implements ChangeMergedListener { }); config.commit(md); - projectCache.evict(config.getProject()); + projectCache.evictAndReindex(config.getProject()); } } diff --git a/java/com/google/gerrit/server/cache/CacheDisplay.java b/java/com/google/gerrit/server/cache/CacheDisplay.java new file mode 100644 index 0000000000..60f5186e8b --- /dev/null +++ b/java/com/google/gerrit/server/cache/CacheDisplay.java @@ -0,0 +1,129 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.cache; + +import com.google.common.base.Strings; +import java.io.IOException; +import java.io.Writer; +import java.util.Collection; + +public class CacheDisplay { + + private final Writer stdout; + private final int nw; + private final Collection<CacheInfo> caches; + + public CacheDisplay(Writer stdout, int nw, Collection<CacheInfo> caches) { + this.stdout = stdout; + this.nw = nw; + this.caches = caches; + } + + public CacheDisplay(Writer stdout, Collection<CacheInfo> caches) { + this(stdout, 30, caches); + } + + public void displayCaches() throws IOException { + stdout.write( + String.format( // + "%1s %-" + nw + "s|%-21s| %-5s |%-9s|\n" // + , + "" // + , + "Name" // + , + "Entries" // + , + "AvgGet" // + , + "Hit Ratio" // + )); + stdout.write( + String.format( // + "%1s %-" + nw + "s|%6s %6s %7s| %-5s |%-4s %-4s|\n" // + , + "" // + , + "" // + , + "Mem" // + , + "Disk" // + , + "Space" // + , + "" // + , + "Mem" // + , + "Disk" // + )); + stdout.write("--"); + for (int i = 0; i < nw; i++) { + stdout.write('-'); + } + stdout.write("+---------------------+---------+---------+\n"); + printMemoryCoreCaches(caches); + printMemoryPluginCaches(caches); + printDiskCaches(caches); + stdout.write('\n'); + } + + private void printMemoryCoreCaches(Collection<CacheInfo> caches) throws IOException { + for (CacheInfo cache : caches) { + if (!cache.name.contains("-") && CacheInfo.CacheType.MEM.equals(cache.type)) { + printCache(cache); + } + } + } + + private void printMemoryPluginCaches(Collection<CacheInfo> caches) throws IOException { + for (CacheInfo cache : caches) { + if (cache.name.contains("-") && CacheInfo.CacheType.MEM.equals(cache.type)) { + printCache(cache); + } + } + } + + private void printDiskCaches(Collection<CacheInfo> caches) throws IOException { + for (CacheInfo cache : caches) { + if (CacheInfo.CacheType.DISK.equals(cache.type)) { + printCache(cache); + } + } + } + + private void printCache(CacheInfo cache) throws IOException { + stdout.write( + String.format( + "%1s %-" + nw + "s|%6s %6s %7s| %7s |%4s %4s|\n", + CacheInfo.CacheType.DISK.equals(cache.type) ? "D" : "", + cache.name, + nullToEmpty(cache.entries.mem), + nullToEmpty(cache.entries.disk), + Strings.nullToEmpty(cache.entries.space), + Strings.nullToEmpty(cache.averageGet), + formatAsPercent(cache.hitRatio.mem), + formatAsPercent(cache.hitRatio.disk))); + } + + private static String nullToEmpty(Long l) { + return l != null ? String.valueOf(l) : ""; + } + + private static String formatAsPercent(Integer i) { + return i != null ? i + "%" : ""; + } +} diff --git a/java/com/google/gerrit/server/cache/CacheInfo.java b/java/com/google/gerrit/server/cache/CacheInfo.java new file mode 100644 index 0000000000..d6eb065339 --- /dev/null +++ b/java/com/google/gerrit/server/cache/CacheInfo.java @@ -0,0 +1,133 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.cache; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheStats; + +public class CacheInfo { + + public String name; + public CacheType type; + public EntriesInfo entries; + public String averageGet; + public HitRatioInfo hitRatio; + + public CacheInfo(Cache<?, ?> cache) { + this(null, cache); + } + + public CacheInfo(String name, Cache<?, ?> cache) { + this.name = name; + + CacheStats stat = cache.stats(); + + entries = new EntriesInfo(); + entries.setMem(cache.size()); + + averageGet = duration(stat.averageLoadPenalty()); + + hitRatio = new HitRatioInfo(); + hitRatio.setMem(stat.hitCount(), stat.requestCount()); + + if (cache instanceof PersistentCache) { + type = CacheType.DISK; + PersistentCache.DiskStats diskStats = ((PersistentCache) cache).diskStats(); + entries.setDisk(diskStats.size()); + entries.setSpace(diskStats.space()); + hitRatio.setDisk(diskStats.hitCount(), diskStats.requestCount()); + } else { + type = CacheType.MEM; + } + } + + private static String duration(double ns) { + if (ns < 0.5) { + return null; + } + String suffix = "ns"; + if (ns >= 1000.0) { + ns /= 1000.0; + suffix = "us"; + } + if (ns >= 1000.0) { + ns /= 1000.0; + suffix = "ms"; + } + if (ns >= 1000.0) { + ns /= 1000.0; + suffix = "s"; + } + return String.format("%4.1f%s", ns, suffix).trim(); + } + + public static class EntriesInfo { + public Long mem; + public Long disk; + public String space; + + public void setMem(long mem) { + this.mem = mem != 0 ? mem : null; + } + + public void setDisk(long disk) { + this.disk = disk != 0 ? disk : null; + } + + public void setSpace(double value) { + space = bytes(value); + } + + private static String bytes(double value) { + value /= 1024; + String suffix = "k"; + + if (value > 1024) { + value /= 1024; + suffix = "m"; + } + if (value > 1024) { + value /= 1024; + suffix = "g"; + } + return String.format("%1$6.2f%2$s", value, suffix).trim(); + } + } + + public static class HitRatioInfo { + public Integer mem; + public Integer disk; + + public void setMem(long value, long total) { + mem = percent(value, total); + } + + public void setDisk(long value, long total) { + disk = percent(value, total); + } + + private static Integer percent(long value, long total) { + if (total <= 0) { + return null; + } + return (int) ((100 * value) / total); + } + } + + public enum CacheType { + MEM, + DISK + } +} diff --git a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java index 6d234ac7ec..324be5e4c3 100644 --- a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java +++ b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java @@ -3096,7 +3096,7 @@ class ReceiveCommits { } if (isConfig(cmd)) { logger.atFine().log("Reloading project in cache"); - projectCache.evict(project); + projectCache.evictAndReindex(project); ProjectState ps = projectCache.get(project.getNameKey()).orElseThrow(illegalState(project.getNameKey())); try { diff --git a/java/com/google/gerrit/server/group/db/RenameGroupOp.java b/java/com/google/gerrit/server/group/db/RenameGroupOp.java index 843b346684..257bc16d17 100644 --- a/java/com/google/gerrit/server/group/db/RenameGroupOp.java +++ b/java/com/google/gerrit/server/group/db/RenameGroupOp.java @@ -123,7 +123,7 @@ class RenameGroupOp extends DefaultQueueOp { // GroupReference ref = config.getGroup(uuid); if (ref == null || newName.equals(ref.getName())) { - projectCache.evict(config.getProject()); + projectCache.evictAndReindex(config.getProject()); return; } @@ -132,7 +132,7 @@ class RenameGroupOp extends DefaultQueueOp { md.setMessage("Rename group " + oldName + " to " + newName + "\n"); try { config.commit(md); - projectCache.evict(config.getProject()); + projectCache.evictAndReindex(config.getProject()); success = true; } catch (IOException e) { logger.atSevere().withCause(e).log( diff --git a/java/com/google/gerrit/server/index/IndexModule.java b/java/com/google/gerrit/server/index/IndexModule.java index 17665c0903..8b04c8dede 100644 --- a/java/com/google/gerrit/server/index/IndexModule.java +++ b/java/com/google/gerrit/server/index/IndexModule.java @@ -51,6 +51,7 @@ import com.google.gerrit.server.index.group.GroupIndexRewriter; import com.google.gerrit.server.index.group.GroupIndexer; import com.google.gerrit.server.index.group.GroupIndexerImpl; import com.google.gerrit.server.index.group.GroupSchemaDefinitions; +import com.google.gerrit.server.index.options.IsFirstInsertForEntry; import com.google.gerrit.server.index.project.ProjectIndexDefinition; import com.google.gerrit.server.index.project.ProjectIndexerImpl; import com.google.inject.Inject; @@ -59,6 +60,7 @@ import com.google.inject.Key; import com.google.inject.Provides; import com.google.inject.ProvisionException; import com.google.inject.Singleton; +import com.google.inject.multibindings.OptionalBinder; import java.util.Collection; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -143,6 +145,9 @@ public class IndexModule extends LifecycleModule { } DynamicSet.setOf(binder(), OnlineUpgradeListener.class); + OptionalBinder.newOptionalBinder(binder(), IsFirstInsertForEntry.class) + .setDefault() + .toInstance(IsFirstInsertForEntry.NO); } @Provides diff --git a/java/com/google/gerrit/server/index/account/AllAccountsIndexer.java b/java/com/google/gerrit/server/index/account/AllAccountsIndexer.java index 63889b798d..ec27db0f1f 100644 --- a/java/com/google/gerrit/server/index/account/AllAccountsIndexer.java +++ b/java/com/google/gerrit/server/index/account/AllAccountsIndexer.java @@ -27,6 +27,7 @@ import com.google.gerrit.server.account.AccountCache; import com.google.gerrit.server.account.AccountState; import com.google.gerrit.server.account.Accounts; import com.google.gerrit.server.index.IndexExecutor; +import com.google.gerrit.server.index.options.IsFirstInsertForEntry; import com.google.inject.Inject; import com.google.inject.Singleton; import java.io.IOException; @@ -50,15 +51,18 @@ public class AllAccountsIndexer extends SiteIndexer<Account.Id, AccountState, Ac private final ListeningExecutorService executor; private final Accounts accounts; private final AccountCache accountCache; + private final IsFirstInsertForEntry isFirstInsertForEntry; @Inject AllAccountsIndexer( @IndexExecutor(BATCH) ListeningExecutorService executor, Accounts accounts, - AccountCache accountCache) { + AccountCache accountCache, + IsFirstInsertForEntry isFirstInsertForEntry) { this.executor = executor; this.accounts = accounts; this.accountCache = accountCache; + this.isFirstInsertForEntry = isFirstInsertForEntry; } @Override @@ -92,7 +96,11 @@ public class AllAccountsIndexer extends SiteIndexer<Account.Id, AccountState, Ac try { Optional<AccountState> a = accountCache.get(id); if (a.isPresent()) { - index.replace(a.get()); + if (isFirstInsertForEntry.equals(isFirstInsertForEntry.YES)) { + index.insert(a.get()); + } else { + index.replace(a.get()); + } } else { index.delete(id); } diff --git a/java/com/google/gerrit/server/index/change/ChangeIndexer.java b/java/com/google/gerrit/server/index/change/ChangeIndexer.java index e0f6bec8e1..a088af02e0 100644 --- a/java/com/google/gerrit/server/index/change/ChangeIndexer.java +++ b/java/com/google/gerrit/server/index/change/ChangeIndexer.java @@ -30,6 +30,7 @@ import com.google.gerrit.index.Index; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.index.IndexExecutor; import com.google.gerrit.server.index.StalenessCheckResult; +import com.google.gerrit.server.index.options.IsFirstInsertForEntry; import com.google.gerrit.server.logging.Metadata; import com.google.gerrit.server.logging.TraceContext; import com.google.gerrit.server.logging.TraceContext.TraceTimer; @@ -78,6 +79,7 @@ public class ChangeIndexer { private final PluginSetContext<ChangeIndexedListener> indexedListeners; private final StalenessChecker stalenessChecker; private final boolean autoReindexIfStale; + private final IsFirstInsertForEntry isFirstInsertForEntry; private final Set<IndexTask> queuedIndexTasks = Collections.newSetFromMap(new ConcurrentHashMap<>()); @@ -94,7 +96,8 @@ public class ChangeIndexer { StalenessChecker stalenessChecker, @IndexExecutor(BATCH) ListeningExecutorService batchExecutor, @Assisted ListeningExecutorService executor, - @Assisted ChangeIndex index) { + @Assisted ChangeIndex index, + IsFirstInsertForEntry isFirstInsertForEntry) { this.executor = executor; this.changeDataFactory = changeDataFactory; this.notesFactory = notesFactory; @@ -105,6 +108,7 @@ public class ChangeIndexer { this.autoReindexIfStale = autoReindexIfStale(cfg); this.index = index; this.indexes = null; + this.isFirstInsertForEntry = isFirstInsertForEntry; } @AssistedInject @@ -117,7 +121,8 @@ public class ChangeIndexer { StalenessChecker stalenessChecker, @IndexExecutor(BATCH) ListeningExecutorService batchExecutor, @Assisted ListeningExecutorService executor, - @Assisted ChangeIndexCollection indexes) { + @Assisted ChangeIndexCollection indexes, + IsFirstInsertForEntry isFirstInsertForEntry) { this.executor = executor; this.changeDataFactory = changeDataFactory; this.notesFactory = notesFactory; @@ -128,6 +133,7 @@ public class ChangeIndexer { this.autoReindexIfStale = autoReindexIfStale(cfg); this.index = null; this.indexes = indexes; + this.isFirstInsertForEntry = isFirstInsertForEntry; } private static boolean autoReindexIfStale(Config cfg) { @@ -198,21 +204,25 @@ public class ChangeIndexer { } private void indexImpl(ChangeData cd) { - logger.atFine().log("Replace change %d in index.", cd.getId().get()); + logger.atFine().log("Reindex change %d in index.", cd.getId().get()); for (Index<?, ChangeData> i : getWriteIndexes()) { try (TraceTimer traceTimer = TraceContext.newTimer( - "Replacing change in index", + "Reindexing change in index", Metadata.builder() .changeId(cd.getId().get()) .patchSetId(cd.currentPatchSet().number()) .indexVersion(i.getSchema().getVersion()) .build())) { - i.replace(cd); + if (isFirstInsertForEntry.equals(isFirstInsertForEntry.YES)) { + i.insert(cd); + } else { + i.replace(cd); + } } catch (RuntimeException e) { throw new StorageException( String.format( - "Failed to replace change %d in index version %d (current patch set = %d)", + "Failed to reindex change %d in index version %d (current patch set = %d)", cd.getId().get(), i.getSchema().getVersion(), cd.currentPatchSet().number()), e); } diff --git a/java/com/google/gerrit/server/index/group/AllGroupsIndexer.java b/java/com/google/gerrit/server/index/group/AllGroupsIndexer.java index 2d77f61e74..fcf0d6d04e 100644 --- a/java/com/google/gerrit/server/index/group/AllGroupsIndexer.java +++ b/java/com/google/gerrit/server/index/group/AllGroupsIndexer.java @@ -30,6 +30,7 @@ import com.google.gerrit.server.group.InternalGroup; import com.google.gerrit.server.group.db.Groups; import com.google.gerrit.server.group.db.GroupsNoteDbConsistencyChecker; import com.google.gerrit.server.index.IndexExecutor; +import com.google.gerrit.server.index.options.IsFirstInsertForEntry; import com.google.inject.Inject; import com.google.inject.Singleton; import java.io.IOException; @@ -54,15 +55,18 @@ public class AllGroupsIndexer extends SiteIndexer<AccountGroup.UUID, InternalGro private final ListeningExecutorService executor; private final GroupCache groupCache; private final Groups groups; + private final IsFirstInsertForEntry isFirstInsertForEntry; @Inject AllGroupsIndexer( @IndexExecutor(BATCH) ListeningExecutorService executor, GroupCache groupCache, - Groups groups) { + Groups groups, + IsFirstInsertForEntry isFirstInsertForEntry) { this.executor = executor; this.groupCache = groupCache; this.groups = groups; + this.isFirstInsertForEntry = isFirstInsertForEntry; } @Override @@ -97,7 +101,11 @@ public class AllGroupsIndexer extends SiteIndexer<AccountGroup.UUID, InternalGro groupCache.evict(uuid); Optional<InternalGroup> internalGroup = groupCache.get(uuid); if (internalGroup.isPresent()) { - index.replace(internalGroup.get()); + if (isFirstInsertForEntry.equals(isFirstInsertForEntry.YES)) { + index.insert(internalGroup.get()); + } else { + index.replace(internalGroup.get()); + } } else { index.delete(uuid); diff --git a/java/com/google/gerrit/server/index/options/IsFirstInsertForEntry.java b/java/com/google/gerrit/server/index/options/IsFirstInsertForEntry.java new file mode 100644 index 0000000000..f943309e07 --- /dev/null +++ b/java/com/google/gerrit/server/index/options/IsFirstInsertForEntry.java @@ -0,0 +1,26 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.options; + +/** + * This enum should be injected and checked to decide on which operation ({@link + * com.google.gerrit.index.Index#replace(Object) replace()} or {@link + * com.google.gerrit.index.Index#insert(Object) insert()}) should be performed on a specific {@link + * com.google.gerrit.index.Index index}. + */ +public enum IsFirstInsertForEntry { + YES, + NO +} diff --git a/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java b/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java index 0e4b688d6c..86c7e94230 100644 --- a/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java +++ b/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java @@ -27,6 +27,7 @@ import com.google.gerrit.index.SiteIndexer; import com.google.gerrit.index.project.ProjectData; import com.google.gerrit.index.project.ProjectIndex; import com.google.gerrit.server.index.IndexExecutor; +import com.google.gerrit.server.index.options.IsFirstInsertForEntry; import com.google.gerrit.server.project.ProjectCache; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -48,12 +49,16 @@ public class AllProjectsIndexer extends SiteIndexer<Project.NameKey, ProjectData private final ListeningExecutorService executor; private final ProjectCache projectCache; + private final IsFirstInsertForEntry isFirstInsertForEntry; @Inject AllProjectsIndexer( - @IndexExecutor(BATCH) ListeningExecutorService executor, ProjectCache projectCache) { + @IndexExecutor(BATCH) ListeningExecutorService executor, + ProjectCache projectCache, + IsFirstInsertForEntry isFirstInsertForEntry) { this.executor = executor; this.projectCache = projectCache; + this.isFirstInsertForEntry = isFirstInsertForEntry; } @Override @@ -79,8 +84,13 @@ public class AllProjectsIndexer extends SiteIndexer<Project.NameKey, ProjectData () -> { try { projectCache.evict(name); - index.replace( - projectCache.get(name).orElseThrow(illegalState(name)).toProjectData()); + ProjectData projectData = + projectCache.get(name).orElseThrow(illegalState(name)).toProjectData(); + if (isFirstInsertForEntry.equals(isFirstInsertForEntry.YES)) { + index.insert(projectData); + } else { + index.replace(projectData); + } verboseWriter.println("Reindexed " + desc); done.incrementAndGet(); } catch (Exception e) { diff --git a/java/com/google/gerrit/server/project/ProjectCache.java b/java/com/google/gerrit/server/project/ProjectCache.java index cd41ce5135..5124e0ead3 100644 --- a/java/com/google/gerrit/server/project/ProjectCache.java +++ b/java/com/google/gerrit/server/project/ProjectCache.java @@ -59,18 +59,25 @@ public interface ProjectCache { Optional<ProjectState> get(@Nullable Project.NameKey projectName) throws StorageException; /** + * Invalidate the cached information about the given project. + * + * @param p the NameKey of the project that is being evicted + */ + void evict(Project.NameKey p); + + /** * Invalidate the cached information about the given project, and triggers reindexing for it * * @param p project that is being evicted */ - void evict(Project p); + void evictAndReindex(Project p); /** * Invalidate the cached information about the given project, and triggers reindexing for it * * @param p the NameKey of the project that is being evicted */ - void evict(Project.NameKey p); + void evictAndReindex(Project.NameKey p); /** * Remove information about the given project from the cache. It will no longer be returned from diff --git a/java/com/google/gerrit/server/project/ProjectCacheImpl.java b/java/com/google/gerrit/server/project/ProjectCacheImpl.java index 1b11ba215a..5ea95fb9c2 100644 --- a/java/com/google/gerrit/server/project/ProjectCacheImpl.java +++ b/java/com/google/gerrit/server/project/ProjectCacheImpl.java @@ -209,16 +209,21 @@ public class ProjectCacheImpl implements ProjectCache { } @Override - public void evict(Project p) { - evict(p.getNameKey()); - } - - @Override public void evict(Project.NameKey p) { if (p != null) { logger.atFine().log("Evict project '%s'", p.get()); inMemoryProjectCache.invalidate(p); } + } + + @Override + public void evictAndReindex(Project p) { + evictAndReindex(p.getNameKey()); + } + + @Override + public void evictAndReindex(Project.NameKey p) { + evict(p); indexer.get().index(p); } @@ -239,7 +244,7 @@ public class ProjectCacheImpl implements ProjectCache { } finally { listLock.unlock(); } - evict(name); + evictAndReindex(name); } @Override diff --git a/java/com/google/gerrit/server/restapi/config/GetCache.java b/java/com/google/gerrit/server/restapi/config/GetCache.java index 93600ea582..5dd3d3dc48 100644 --- a/java/com/google/gerrit/server/restapi/config/GetCache.java +++ b/java/com/google/gerrit/server/restapi/config/GetCache.java @@ -16,6 +16,7 @@ package com.google.gerrit.server.restapi.config; import com.google.gerrit.extensions.restapi.Response; import com.google.gerrit.extensions.restapi.RestReadView; +import com.google.gerrit.server.cache.CacheInfo; import com.google.gerrit.server.config.CacheResource; import com.google.inject.Singleton; @@ -23,7 +24,7 @@ import com.google.inject.Singleton; public class GetCache implements RestReadView<CacheResource> { @Override - public Response<ListCaches.CacheInfo> apply(CacheResource rsrc) { - return Response.ok(new ListCaches.CacheInfo(rsrc.getName(), rsrc.getCache())); + public Response<CacheInfo> apply(CacheResource rsrc) { + return Response.ok(new CacheInfo(rsrc.getName(), rsrc.getCache())); } } diff --git a/java/com/google/gerrit/server/restapi/config/ListCaches.java b/java/com/google/gerrit/server/restapi/config/ListCaches.java index ccafbe89b1..ffc65c92da 100644 --- a/java/com/google/gerrit/server/restapi/config/ListCaches.java +++ b/java/com/google/gerrit/server/restapi/config/ListCaches.java @@ -22,7 +22,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.stream.Collectors.joining; import com.google.common.cache.Cache; -import com.google.common.cache.CacheStats; import com.google.common.collect.Streams; import com.google.gerrit.extensions.annotations.RequiresAnyCapability; import com.google.gerrit.extensions.registration.DynamicMap; @@ -30,7 +29,7 @@ import com.google.gerrit.extensions.registration.Extension; import com.google.gerrit.extensions.restapi.BinaryResult; import com.google.gerrit.extensions.restapi.Response; import com.google.gerrit.extensions.restapi.RestReadView; -import com.google.gerrit.server.cache.PersistentCache; +import com.google.gerrit.server.cache.CacheInfo; import com.google.gerrit.server.config.ConfigResource; import com.google.inject.Inject; import java.util.Map; @@ -87,118 +86,4 @@ public class ListCaches implements RestReadView<ConfigResource> { } return Response.ok(cacheNames.collect(toImmutableList())); } - - public enum CacheType { - MEM, - DISK - } - - public static class CacheInfo { - public String name; - public CacheType type; - public EntriesInfo entries; - public String averageGet; - public HitRatioInfo hitRatio; - - public CacheInfo(Cache<?, ?> cache) { - this(null, cache); - } - - public CacheInfo(String name, Cache<?, ?> cache) { - this.name = name; - - CacheStats stat = cache.stats(); - - entries = new EntriesInfo(); - entries.setMem(cache.size()); - - averageGet = duration(stat.averageLoadPenalty()); - - hitRatio = new HitRatioInfo(); - hitRatio.setMem(stat.hitCount(), stat.requestCount()); - - if (cache instanceof PersistentCache) { - type = CacheType.DISK; - PersistentCache.DiskStats diskStats = ((PersistentCache) cache).diskStats(); - entries.setDisk(diskStats.size()); - entries.setSpace(diskStats.space()); - hitRatio.setDisk(diskStats.hitCount(), diskStats.requestCount()); - } else { - type = CacheType.MEM; - } - } - - private static String duration(double ns) { - if (ns < 0.5) { - return null; - } - String suffix = "ns"; - if (ns >= 1000.0) { - ns /= 1000.0; - suffix = "us"; - } - if (ns >= 1000.0) { - ns /= 1000.0; - suffix = "ms"; - } - if (ns >= 1000.0) { - ns /= 1000.0; - suffix = "s"; - } - return String.format("%4.1f%s", ns, suffix).trim(); - } - } - - public static class EntriesInfo { - public Long mem; - public Long disk; - public String space; - - public void setMem(long mem) { - this.mem = mem != 0 ? mem : null; - } - - public void setDisk(long disk) { - this.disk = disk != 0 ? disk : null; - } - - public void setSpace(double value) { - space = bytes(value); - } - - private static String bytes(double value) { - value /= 1024; - String suffix = "k"; - - if (value > 1024) { - value /= 1024; - suffix = "m"; - } - if (value > 1024) { - value /= 1024; - suffix = "g"; - } - return String.format("%1$6.2f%2$s", value, suffix).trim(); - } - } - - public static class HitRatioInfo { - public Integer mem; - public Integer disk; - - public void setMem(long value, long total) { - mem = percent(value, total); - } - - public void setDisk(long value, long total) { - disk = percent(value, total); - } - - private static Integer percent(long value, long total) { - if (total <= 0) { - return null; - } - return (int) ((100 * value) / total); - } - } } diff --git a/java/com/google/gerrit/server/restapi/project/CreateLabel.java b/java/com/google/gerrit/server/restapi/project/CreateLabel.java index 3e1ef49420..e93c25ca59 100644 --- a/java/com/google/gerrit/server/restapi/project/CreateLabel.java +++ b/java/com/google/gerrit/server/restapi/project/CreateLabel.java @@ -101,7 +101,7 @@ public class CreateLabel config.commit(md); - projectCache.evict(rsrc.getProjectState().getProject()); + projectCache.evictAndReindex(rsrc.getProjectState().getProject()); return Response.created(LabelDefinitionJson.format(rsrc.getNameKey(), labelType)); } diff --git a/java/com/google/gerrit/server/restapi/project/DeleteLabel.java b/java/com/google/gerrit/server/restapi/project/DeleteLabel.java index 531640cf38..8a1927aea2 100644 --- a/java/com/google/gerrit/server/restapi/project/DeleteLabel.java +++ b/java/com/google/gerrit/server/restapi/project/DeleteLabel.java @@ -90,7 +90,7 @@ public class DeleteLabel implements RestModifyView<LabelResource, InputWithCommi config.commit(md); } - projectCache.evict(rsrc.getProject().getProjectState().getProject()); + projectCache.evictAndReindex(rsrc.getProject().getProjectState().getProject()); return Response.none(); } diff --git a/java/com/google/gerrit/server/restapi/project/GetAccess.java b/java/com/google/gerrit/server/restapi/project/GetAccess.java index a79439c1b4..651e7f078f 100644 --- a/java/com/google/gerrit/server/restapi/project/GetAccess.java +++ b/java/com/google/gerrit/server/restapi/project/GetAccess.java @@ -153,12 +153,12 @@ public class GetAccess implements RestReadView<ProjectResource> { if (config.updateGroupNames(groupBackend)) { md.setMessage("Update group names\n"); config.commit(md); - projectCache.evict(config.getProject()); + projectCache.evictAndReindex(config.getProject()); projectState = projectCache.get(projectName).orElseThrow(illegalState(projectName)); perm = permissionBackend.currentUser().project(projectName); } else if (config.getRevision() != null && !config.getRevision().equals(projectState.getConfig().getRevision().orElse(null))) { - projectCache.evict(config.getProject()); + projectCache.evictAndReindex(config.getProject()); projectState = projectCache.get(projectName).orElseThrow(illegalState(projectName)); perm = permissionBackend.currentUser().project(projectName); } diff --git a/java/com/google/gerrit/server/restapi/project/PostLabels.java b/java/com/google/gerrit/server/restapi/project/PostLabels.java index 0c42ab27ef..b72e10fa11 100644 --- a/java/com/google/gerrit/server/restapi/project/PostLabels.java +++ b/java/com/google/gerrit/server/restapi/project/PostLabels.java @@ -139,7 +139,7 @@ public class PostLabels if (dirty) { config.commit(md); - projectCache.evict(rsrc.getProjectState().getProject()); + projectCache.evictAndReindex(rsrc.getProjectState().getProject()); } } diff --git a/java/com/google/gerrit/server/restapi/project/PutConfig.java b/java/com/google/gerrit/server/restapi/project/PutConfig.java index 55ea312a96..3bf432b835 100644 --- a/java/com/google/gerrit/server/restapi/project/PutConfig.java +++ b/java/com/google/gerrit/server/restapi/project/PutConfig.java @@ -164,7 +164,7 @@ public class PutConfig implements RestModifyView<ProjectResource, ConfigInput> { md.setMessage("Modified project settings\n"); try { projectConfig.commit(md); - projectCache.evict(projectConfig.getProject()); + projectCache.evictAndReindex(projectConfig.getProject()); md.getRepository().setGitwebDescription(projectConfig.getProject().getDescription()); } catch (IOException e) { if (e.getCause() instanceof ConfigInvalidException) { diff --git a/java/com/google/gerrit/server/restapi/project/PutDescription.java b/java/com/google/gerrit/server/restapi/project/PutDescription.java index a65c6260e2..ec42035ffc 100644 --- a/java/com/google/gerrit/server/restapi/project/PutDescription.java +++ b/java/com/google/gerrit/server/restapi/project/PutDescription.java @@ -84,7 +84,7 @@ public class PutDescription implements RestModifyView<ProjectResource, Descripti md.setAuthor(user); md.setMessage(msg); config.commit(md); - cache.evict(resource.getProjectState().getProject()); + cache.evictAndReindex(resource.getProjectState().getProject()); md.getRepository().setGitwebDescription(config.getProject().getDescription()); return Strings.isNullOrEmpty(config.getProject().getDescription()) diff --git a/java/com/google/gerrit/server/restapi/project/SetAccess.java b/java/com/google/gerrit/server/restapi/project/SetAccess.java index 794cae840c..07dbeca490 100644 --- a/java/com/google/gerrit/server/restapi/project/SetAccess.java +++ b/java/com/google/gerrit/server/restapi/project/SetAccess.java @@ -125,7 +125,7 @@ public class SetAccess implements RestModifyView<ProjectResource, ProjectAccessI } config.commit(md); - projectCache.evict(config.getProject()); + projectCache.evictAndReindex(config.getProject()); createGroupPermissionSyncer.syncIfNeeded(); } catch (InvalidNameException e) { throw new BadRequestException(e.toString()); diff --git a/java/com/google/gerrit/server/restapi/project/SetDefaultDashboard.java b/java/com/google/gerrit/server/restapi/project/SetDefaultDashboard.java index 5aef76a30c..853d7dff0e 100644 --- a/java/com/google/gerrit/server/restapi/project/SetDefaultDashboard.java +++ b/java/com/google/gerrit/server/restapi/project/SetDefaultDashboard.java @@ -115,7 +115,7 @@ class SetDefaultDashboard implements RestModifyView<DashboardResource, SetDashbo md.setAuthor(rsrc.getUser().asIdentifiedUser()); md.setMessage(msg); config.commit(md); - cache.evict(rsrc.getProjectState().getProject()); + cache.evictAndReindex(rsrc.getProjectState().getProject()); if (target != null) { Response<DashboardInfo> response = get.get().apply(target); diff --git a/java/com/google/gerrit/server/restapi/project/SetLabel.java b/java/com/google/gerrit/server/restapi/project/SetLabel.java index ffc591b5cd..1e04dc4a80 100644 --- a/java/com/google/gerrit/server/restapi/project/SetLabel.java +++ b/java/com/google/gerrit/server/restapi/project/SetLabel.java @@ -93,7 +93,7 @@ public class SetLabel implements RestModifyView<LabelResource, LabelDefinitionIn config.getLabelSections().get(newName.isEmpty() ? labelType.getName() : newName); config.commit(md); - projectCache.evict(rsrc.getProject().getProjectState().getProject()); + projectCache.evictAndReindex(rsrc.getProject().getProjectState().getProject()); } } return Response.ok(LabelDefinitionJson.format(rsrc.getProject().getNameKey(), labelType)); diff --git a/java/com/google/gerrit/server/restapi/project/SetParent.java b/java/com/google/gerrit/server/restapi/project/SetParent.java index 91c29f5332..ef31dc5b0b 100644 --- a/java/com/google/gerrit/server/restapi/project/SetParent.java +++ b/java/com/google/gerrit/server/restapi/project/SetParent.java @@ -114,7 +114,7 @@ public class SetParent md.setAuthor(user); md.setMessage(msg); config.commit(md); - cache.evict(rsrc.getProjectState().getProject()); + cache.evictAndReindex(rsrc.getProjectState().getProject()); Project.NameKey parent = config.getProject().getParent(allProjects); requireNonNull(parent); diff --git a/java/com/google/gerrit/server/submit/SubmitStrategyOp.java b/java/com/google/gerrit/server/submit/SubmitStrategyOp.java index 3b77dd97e7..dfee2f8356 100644 --- a/java/com/google/gerrit/server/submit/SubmitStrategyOp.java +++ b/java/com/google/gerrit/server/submit/SubmitStrategyOp.java @@ -472,7 +472,7 @@ abstract class SubmitStrategyOp implements BatchUpdateOp { // TODO(dborowitz): Move to BatchUpdate? Would also allow us to run once // per project even if multiple changes to refs/meta/config are submitted. if (RefNames.REFS_CONFIG.equals(getDest().branch())) { - args.projectCache.evict(getProject()); + args.projectCache.evictAndReindex(getProject()); ProjectState p = args.projectCache.get(getProject()).orElseThrow(illegalState(getProject())); try (Repository git = args.repoManager.openRepository(getProject())) { diff --git a/java/com/google/gerrit/sshd/commands/ShowCaches.java b/java/com/google/gerrit/sshd/commands/ShowCaches.java index ba84179085..219ceaab2d 100644 --- a/java/com/google/gerrit/sshd/commands/ShowCaches.java +++ b/java/com/google/gerrit/sshd/commands/ShowCaches.java @@ -24,6 +24,8 @@ import com.google.gerrit.extensions.annotations.RequiresAnyCapability; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.extensions.restapi.AuthException; import com.google.gerrit.server.CurrentUser; +import com.google.gerrit.server.cache.CacheDisplay; +import com.google.gerrit.server.cache.CacheInfo; import com.google.gerrit.server.config.ConfigResource; import com.google.gerrit.server.permissions.GlobalPermission; import com.google.gerrit.server.permissions.PermissionBackend; @@ -35,8 +37,6 @@ import com.google.gerrit.server.restapi.config.GetSummary.SummaryInfo; import com.google.gerrit.server.restapi.config.GetSummary.TaskSummaryInfo; import com.google.gerrit.server.restapi.config.GetSummary.ThreadSummaryInfo; import com.google.gerrit.server.restapi.config.ListCaches; -import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo; -import com.google.gerrit.server.restapi.config.ListCaches.CacheType; import com.google.gerrit.server.util.time.TimeUtil; import com.google.gerrit.sshd.CommandMetaData; import com.google.gerrit.sshd.SshCommand; @@ -123,52 +123,8 @@ final class ShowCaches extends SshCommand { stdout.format("%-25s %-20s uptime %16s\n", "", "", uptime(now.getTime() - serverStarted)); stdout.print('\n'); - stdout.print( - String.format( // - "%1s %-" + nw + "s|%-21s| %-5s |%-9s|\n" // - , - "" // - , - "Name" // - , - "Entries" // - , - "AvgGet" // - , - "Hit Ratio" // - )); - stdout.print( - String.format( // - "%1s %-" + nw + "s|%6s %6s %7s| %-5s |%-4s %-4s|\n" // - , - "" // - , - "" // - , - "Mem" // - , - "Disk" // - , - "Space" // - , - "" // - , - "Mem" // - , - "Disk" // - )); - stdout.print("--"); - for (int i = 0; i < nw; i++) { - stdout.print('-'); - } - stdout.print("+---------------------+---------+---------+\n"); - try { - Collection<CacheInfo> caches = getCaches(); - printMemoryCoreCaches(caches); - printMemoryPluginCaches(caches); - printDiskCaches(caches); - stdout.print('\n'); + new CacheDisplay(stdout, nw, getCaches()).displayCaches(); boolean showJvm; try { @@ -209,52 +165,6 @@ final class ShowCaches extends SshCommand { return caches.values(); } - private void printMemoryCoreCaches(Collection<CacheInfo> caches) { - for (CacheInfo cache : caches) { - if (!cache.name.contains("-") && CacheType.MEM.equals(cache.type)) { - printCache(cache); - } - } - } - - private void printMemoryPluginCaches(Collection<CacheInfo> caches) { - for (CacheInfo cache : caches) { - if (cache.name.contains("-") && CacheType.MEM.equals(cache.type)) { - printCache(cache); - } - } - } - - private void printDiskCaches(Collection<CacheInfo> caches) { - for (CacheInfo cache : caches) { - if (CacheType.DISK.equals(cache.type)) { - printCache(cache); - } - } - } - - private void printCache(CacheInfo cache) { - stdout.print( - String.format( - "%1s %-" + nw + "s|%6s %6s %7s| %7s |%4s %4s|\n", - CacheType.DISK.equals(cache.type) ? "D" : "", - cache.name, - nullToEmpty(cache.entries.mem), - nullToEmpty(cache.entries.disk), - Strings.nullToEmpty(cache.entries.space), - Strings.nullToEmpty(cache.averageGet), - formatAsPercent(cache.hitRatio.mem), - formatAsPercent(cache.hitRatio.disk))); - } - - private static String nullToEmpty(Long l) { - return l != null ? String.valueOf(l) : ""; - } - - private static String formatAsPercent(Integer i) { - return i != null ? String.valueOf(i) + "%" : ""; - } - private void memSummary(MemSummaryInfo memSummary) { stdout.format( "Mem: %s total = %s used + %s free + %s buffers\n", diff --git a/javatests/com/google/gerrit/acceptance/ProjectResetterTest.java b/javatests/com/google/gerrit/acceptance/ProjectResetterTest.java index ff9bac9afb..7d04558561 100644 --- a/javatests/com/google/gerrit/acceptance/ProjectResetterTest.java +++ b/javatests/com/google/gerrit/acceptance/ProjectResetterTest.java @@ -231,7 +231,7 @@ public class ProjectResetterTest { updateRef(repo2, metaConfig); } - verify(projectCache, only()).evict(project2); + verify(projectCache, only()).evictAndReindex(project2); } @Test @@ -248,7 +248,7 @@ public class ProjectResetterTest { createRef(repo2, RefNames.REFS_CONFIG); } - verify(projectCache, only()).evict(project2); + verify(projectCache, only()).evictAndReindex(project2); } @Test diff --git a/javatests/com/google/gerrit/acceptance/api/accounts/AgreementsIT.java b/javatests/com/google/gerrit/acceptance/api/accounts/AgreementsIT.java index 4c3c77f12b..24b7998724 100644 --- a/javatests/com/google/gerrit/acceptance/api/accounts/AgreementsIT.java +++ b/javatests/com/google/gerrit/acceptance/api/accounts/AgreementsIT.java @@ -71,7 +71,7 @@ public class AgreementsIT extends AbstractDaemonTest { config.updateProject( p -> p.setBooleanConfig(BooleanProjectConfig.USE_CONTRIBUTOR_AGREEMENTS, value)); config.commit(md); - projectCache.evict(config.getProject()); + projectCache.evictAndReindex(config.getProject()); } } diff --git a/javatests/com/google/gerrit/acceptance/api/project/ProjectIT.java b/javatests/com/google/gerrit/acceptance/api/project/ProjectIT.java index d5fc1c1af4..2847f64bd6 100644 --- a/javatests/com/google/gerrit/acceptance/api/project/ProjectIT.java +++ b/javatests/com/google/gerrit/acceptance/api/project/ProjectIT.java @@ -970,7 +970,7 @@ public class ProjectIT extends AbstractDaemonTest { ProjectConfig config = projectConfigFactory.read(md); config.renameGroup(AccountGroup.uuid(group.id), newName); config.commit(md); - projectCache.evict(config.getProject()); + projectCache.evictAndReindex(config.getProject()); } Optional<String> afterRename = diff --git a/javatests/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java b/javatests/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java index 415aa79558..c3bcbd3a5a 100644 --- a/javatests/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java +++ b/javatests/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java @@ -226,7 +226,7 @@ public abstract class AbstractSubmoduleSubscription extends AbstractDaemonTest { ObjectId oldId = pc.getRevision(); ObjectId newId = pc.commit(md); assertThat(newId).isNotEqualTo(oldId); - projectCache.evict(pc.getProject()); + projectCache.evictAndReindex(pc.getProject()); } } diff --git a/javatests/com/google/gerrit/acceptance/rest/config/CacheOperationsIT.java b/javatests/com/google/gerrit/acceptance/rest/config/CacheOperationsIT.java index daeb032a82..e4b4e4ad96 100644 --- a/javatests/com/google/gerrit/acceptance/rest/config/CacheOperationsIT.java +++ b/javatests/com/google/gerrit/acceptance/rest/config/CacheOperationsIT.java @@ -25,7 +25,7 @@ import com.google.gerrit.acceptance.AbstractDaemonTest; import com.google.gerrit.acceptance.RestResponse; import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; import com.google.gerrit.common.data.GlobalCapability; -import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo; +import com.google.gerrit.server.cache.CacheInfo; import com.google.gerrit.server.restapi.config.PostCaches; import com.google.inject.Inject; import java.util.Arrays; diff --git a/javatests/com/google/gerrit/acceptance/rest/config/FlushCacheIT.java b/javatests/com/google/gerrit/acceptance/rest/config/FlushCacheIT.java index a161ec4eb7..164f683f22 100644 --- a/javatests/com/google/gerrit/acceptance/rest/config/FlushCacheIT.java +++ b/javatests/com/google/gerrit/acceptance/rest/config/FlushCacheIT.java @@ -23,7 +23,7 @@ import com.google.gerrit.acceptance.AbstractDaemonTest; import com.google.gerrit.acceptance.RestResponse; import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; import com.google.gerrit.common.data.GlobalCapability; -import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo; +import com.google.gerrit.server.cache.CacheInfo; import com.google.inject.Inject; import org.junit.Test; diff --git a/javatests/com/google/gerrit/acceptance/rest/config/GetCacheIT.java b/javatests/com/google/gerrit/acceptance/rest/config/GetCacheIT.java index 247d63b797..8765360423 100644 --- a/javatests/com/google/gerrit/acceptance/rest/config/GetCacheIT.java +++ b/javatests/com/google/gerrit/acceptance/rest/config/GetCacheIT.java @@ -18,8 +18,7 @@ import static com.google.common.truth.Truth.assertThat; import com.google.gerrit.acceptance.AbstractDaemonTest; import com.google.gerrit.acceptance.RestResponse; -import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo; -import com.google.gerrit.server.restapi.config.ListCaches.CacheType; +import com.google.gerrit.server.cache.CacheInfo; import org.junit.Test; public class GetCacheIT extends AbstractDaemonTest { @@ -31,7 +30,7 @@ public class GetCacheIT extends AbstractDaemonTest { CacheInfo result = newGson().fromJson(r.getReader(), CacheInfo.class); assertThat(result.name).isEqualTo("accounts"); - assertThat(result.type).isEqualTo(CacheType.MEM); + assertThat(result.type).isEqualTo(CacheInfo.CacheType.MEM); assertThat(result.entries.mem).isAtLeast(1L); assertThat(result.averageGet).isNotNull(); assertThat(result.averageGet).endsWith("s"); diff --git a/javatests/com/google/gerrit/acceptance/rest/config/ListCachesIT.java b/javatests/com/google/gerrit/acceptance/rest/config/ListCachesIT.java index 8baeffc4aa..be21436864 100644 --- a/javatests/com/google/gerrit/acceptance/rest/config/ListCachesIT.java +++ b/javatests/com/google/gerrit/acceptance/rest/config/ListCachesIT.java @@ -21,8 +21,7 @@ import com.google.common.collect.Ordering; import com.google.common.io.BaseEncoding; import com.google.gerrit.acceptance.AbstractDaemonTest; import com.google.gerrit.acceptance.RestResponse; -import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo; -import com.google.gerrit.server.restapi.config.ListCaches.CacheType; +import com.google.gerrit.server.cache.CacheInfo; import com.google.gson.reflect.TypeToken; import java.util.Arrays; import java.util.List; @@ -40,7 +39,7 @@ public class ListCachesIT extends AbstractDaemonTest { assertThat(result).containsKey("accounts"); CacheInfo accountsCacheInfo = result.get("accounts"); - assertThat(accountsCacheInfo.type).isEqualTo(CacheType.MEM); + assertThat(accountsCacheInfo.type).isEqualTo(CacheInfo.CacheType.MEM); assertThat(accountsCacheInfo.entries.mem).isAtLeast(1L); assertThat(accountsCacheInfo.averageGet).isNotNull(); assertThat(accountsCacheInfo.averageGet).endsWith("s"); diff --git a/javatests/com/google/gerrit/acceptance/rest/project/AccessIT.java b/javatests/com/google/gerrit/acceptance/rest/project/AccessIT.java index caf9b907e7..908f17aab9 100644 --- a/javatests/com/google/gerrit/acceptance/rest/project/AccessIT.java +++ b/javatests/com/google/gerrit/acceptance/rest/project/AccessIT.java @@ -177,7 +177,7 @@ public class AccessIT extends AbstractDaemonTest { ProjectAccessInfo expected = pApi().access(); grantRevertPermission.execute(newProjectName); - projectCache.evict(newProjectName); + projectCache.evictAndReindex(newProjectName); ProjectAccessInfo actual = pApi().access(); // Permissions don't change assertThat(expected.local).isEqualTo(actual.local); diff --git a/javatests/com/google/gerrit/server/change/LabelNormalizerTest.java b/javatests/com/google/gerrit/server/change/LabelNormalizerTest.java index 683f5a624c..e5f3e2391d 100644 --- a/javatests/com/google/gerrit/server/change/LabelNormalizerTest.java +++ b/javatests/com/google/gerrit/server/change/LabelNormalizerTest.java @@ -191,7 +191,7 @@ public class LabelNormalizerTest { private void save(ProjectConfig pc) throws Exception { try (MetaDataUpdate md = metaDataUpdateFactory.create(pc.getProject().getNameKey(), user)) { pc.commit(md); - projectCache.evict(pc.getProject().getNameKey()); + projectCache.evictAndReindex(pc.getProject().getNameKey()); } } diff --git a/javatests/com/google/gerrit/server/index/change/FakeChangeIndex.java b/javatests/com/google/gerrit/server/index/change/FakeChangeIndex.java index a23ccaba9a..fc56a3c00f 100644 --- a/javatests/com/google/gerrit/server/index/change/FakeChangeIndex.java +++ b/javatests/com/google/gerrit/server/index/change/FakeChangeIndex.java @@ -74,6 +74,11 @@ public class FakeChangeIndex implements ChangeIndex { } @Override + public void insert(ChangeData obj) { + throw new UnsupportedOperationException(); + } + + @Override public void replace(ChangeData cd) { throw new UnsupportedOperationException(); } diff --git a/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java b/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java index 297977c742..88dda0ff51 100644 --- a/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java +++ b/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java @@ -1030,7 +1030,7 @@ public abstract class AbstractQueryChangesTest extends GerritServerTests { cfg.upsertLabelType(verified); cfg.commit(md); } - projectCache.evict(project); + projectCache.evictAndReindex(project); String heads = RefNames.REFS_HEADS + "*"; projectOperations @@ -1871,7 +1871,7 @@ public abstract class AbstractQueryChangesTest extends GerritServerTests { }); config.commit(md); - projectCache.evict(config.getProject()); + projectCache.evictAndReindex(config.getProject()); } } diff --git a/plugins/delete-project b/plugins/delete-project -Subproject 60ce67dd53ad64c33a2c34aae31e9ee82397910 +Subproject 7d060dab5f311ec498f9a3b0aa58ce797b8c4d2 diff --git a/plugins/webhooks b/plugins/webhooks -Subproject 9fc9c2d4e69f7e2701cbcd873977d3312a231a8 +Subproject 83dda664bf0ad96249dfd99a03b11d2eaf32703 |