1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.project;
import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
import static com.google.gerrit.server.project.ProjectCache.illegalState;
import com.google.common.base.Stopwatch;
import com.google.common.flogger.FluentLogger;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.gerrit.entities.Project;
import com.google.gerrit.index.SiteIndexer;
import com.google.gerrit.index.project.ProjectData;
import com.google.gerrit.index.project.ProjectIndex;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.gerrit.server.project.ProjectCache;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.TextProgressMonitor;
/**
* Implementation that can index all projects on a host. Used by Gerrit's initialization and upgrade
* programs as well as by REST API endpoints that offer this functionality.
*/
@Singleton
public class AllProjectsIndexer extends SiteIndexer<Project.NameKey, ProjectData, ProjectIndex> {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final ListeningExecutorService executor;
private final ProjectCache projectCache;
private final IsFirstInsertForEntry isFirstInsertForEntry;
@Inject
AllProjectsIndexer(
@IndexExecutor(BATCH) ListeningExecutorService executor,
ProjectCache projectCache,
IsFirstInsertForEntry isFirstInsertForEntry) {
this.executor = executor;
this.projectCache = projectCache;
this.isFirstInsertForEntry = isFirstInsertForEntry;
}
@Override
public SiteIndexer.Result indexAll(final ProjectIndex index) {
ProgressMonitor progress = new TextProgressMonitor(newPrintWriter(progressOut));
progress.start(2);
List<Project.NameKey> names = collectProjects(progress);
return reindexProjects(index, names, progress);
}
private SiteIndexer.Result reindexProjects(
ProjectIndex index, List<Project.NameKey> names, ProgressMonitor progress) {
progress.beginTask("Reindexing projects", names.size());
List<ListenableFuture<?>> futures = new ArrayList<>(names.size());
AtomicBoolean ok = new AtomicBoolean(true);
AtomicInteger done = new AtomicInteger();
AtomicInteger failed = new AtomicInteger();
Stopwatch sw = Stopwatch.createStarted();
for (Project.NameKey name : names) {
String desc = "project " + name;
ListenableFuture<?> future =
executor.submit(
() -> {
try {
projectCache.evict(name);
ProjectData projectData =
projectCache.get(name).orElseThrow(illegalState(name)).toProjectData();
if (isFirstInsertForEntry.equals(isFirstInsertForEntry.YES)) {
index.insert(projectData);
} else {
index.replace(projectData);
}
verboseWriter.println("Reindexed " + desc);
done.incrementAndGet();
} catch (Exception e) {
failed.incrementAndGet();
throw e;
}
return null;
});
addErrorListener(future, desc, progress, ok);
futures.add(future);
}
try {
Futures.successfulAsList(futures).get();
} catch (ExecutionException | InterruptedException e) {
logger.atSevere().withCause(e).log("Error waiting on project futures");
return SiteIndexer.Result.create(sw, false, 0, 0);
}
progress.endTask();
return SiteIndexer.Result.create(sw, ok.get(), done.get(), failed.get());
}
private List<Project.NameKey> collectProjects(ProgressMonitor progress) {
progress.beginTask("Collecting projects", ProgressMonitor.UNKNOWN);
List<Project.NameKey> names = new ArrayList<>();
for (Project.NameKey nameKey : projectCache.all()) {
names.add(nameKey);
}
progress.endTask();
return names;
}
}
|