diff options
28 files changed, 558 insertions, 336 deletions
@@ -34,12 +34,7 @@ java_test( source_under_test = [':replication__plugin'], deps = [ ':replication__plugin', - '//gerrit-common:server', - '//gerrit-reviewdb:server', - '//gerrit-server:server', - '//lib:gwtorm', - '//lib:junit', - '//lib/easymock:easymock', - '//lib/jgit:jgit', + '//gerrit-acceptance-framework:lib', + '//gerrit-plugin-api:lib', ], ) diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java index 2743549..22b29b1 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java @@ -49,13 +49,16 @@ public class AutoReloadConfigDecorator implements ReplicationConfig { private final GroupIncludeCache groupIncludeCache; @Inject - public AutoReloadConfigDecorator(Injector injector, SitePaths site, - RemoteSiteUser.Factory ruf, PluginUser pu, - GitRepositoryManager grm, GroupBackend gb, + public AutoReloadConfigDecorator(Injector injector, + SitePaths site, + RemoteSiteUser.Factory ruf, + PluginUser pu, + GitRepositoryManager grm, + GroupBackend gb, WorkQueue workQueue, ReplicationStateListener stateLog, - GroupIncludeCache groupIncludeCache) throws ConfigInvalidException, - IOException { + GroupIncludeCache groupIncludeCache) + throws ConfigInvalidException, IOException { this.injector = injector; this.site = site; this.remoteSiteUserFactory = ruf; diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java index 9ce4a54..e960533 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java @@ -13,7 +13,7 @@ // limitations under the License. package com.googlesource.gerrit.plugins.replication; -interface CredentialsFactory { +public interface CredentialsFactory { SecureCredentialsProvider create(String remoteName); diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java index e267da3..8164c09 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java @@ -14,8 +14,9 @@ package com.googlesource.gerrit.plugins.replication; -import com.google.common.base.MoreObjects; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet.Builder; import com.google.common.collect.Lists; @@ -31,7 +32,6 @@ import com.google.gerrit.server.account.GroupBackend; import com.google.gerrit.server.account.GroupBackends; import com.google.gerrit.server.account.GroupIncludeCache; import com.google.gerrit.server.account.ListGroupMembership; -import com.google.gerrit.server.config.ConfigUtil; import com.google.gerrit.server.config.RequestScopedReviewDbProvider; import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.git.PerThreadRequestScope; @@ -47,7 +47,6 @@ import com.google.inject.assistedinject.FactoryModuleBuilder; import com.google.inject.servlet.RequestScoped; import org.apache.commons.io.FilenameUtils; -import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; @@ -59,83 +58,56 @@ import org.slf4j.Logger; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; -class Destination { +public class Destination { private static final Logger repLog = ReplicationQueue.repLog; private final ReplicationStateListener stateLog; - - private final int poolThreads; - private final String poolName; - - private final RemoteConfig remote; - private final String[] adminUrls; - private final String[] urls; - private final String[] projects; - private final String[] authGroupNames; - private final int delay; - private final int retryDelay; private final Object stateLock = new Object(); - private final int lockErrorMaxRetries; private final Map<URIish, PushOne> pending = new HashMap<>(); private final Map<URIish, PushOne> inFlight = new HashMap<>(); private final PushOne.Factory opFactory; private final ProjectControl.Factory projectControlFactory; private final GitRepositoryManager gitManager; - private final boolean createMissingRepos; - private final boolean replicatePermissions; - private final boolean replicateProjectDeletions; - private final String remoteNameStyle; private volatile WorkQueue.Executor pool; private final PerThreadRequestScope.Scoper threadScoper; + private final DestinationConfiguration config; - protected static enum RetryReason { + protected enum RetryReason { TRANSPORT_ERROR, COLLISION, REPOSITORY_MISSING; } - Destination(final Injector injector, - final RemoteConfig rc, - final Config cfg, - final RemoteSiteUser.Factory replicationUserFactory, - final PluginUser pluginUser, - final GitRepositoryManager gitRepositoryManager, - final GroupBackend groupBackend, - final ReplicationStateListener stateLog, - final GroupIncludeCache groupIncludeCache) { - remote = rc; + public static class QueueInfo { + public final Map<URIish, PushOne> pending; + public final Map<URIish, PushOne> inFlight; + + public QueueInfo(Map<URIish, PushOne> pending, + Map<URIish, PushOne> inFlight) { + this.pending = ImmutableMap.copyOf(pending); + this.inFlight = ImmutableMap.copyOf(inFlight); + } + } + + protected Destination(Injector injector, + DestinationConfiguration cfg, + RemoteSiteUser.Factory replicationUserFactory, + PluginUser pluginUser, + GitRepositoryManager gitRepositoryManager, + GroupBackend groupBackend, + ReplicationStateListener stateLog, + GroupIncludeCache groupIncludeCache) { + config = cfg; gitManager = gitRepositoryManager; this.stateLog = stateLog; - delay = Math.max(0, - getTimeUnit(rc, cfg, "replicationdelay", 15, TimeUnit.SECONDS)); - retryDelay = Math.max(0, - getTimeUnit(rc, cfg, "replicationretry", 1, TimeUnit.MINUTES)); - lockErrorMaxRetries = cfg.getInt("replication", "lockErrorMaxRetries", 0); - adminUrls = cfg.getStringList("remote", rc.getName(), "adminUrl"); - urls = cfg.getStringList("remote", rc.getName(), "url"); - - poolThreads = Math.max(0, getInt(rc, cfg, "threads", 1)); - poolName = "ReplicateTo-" + rc.getName(); - createMissingRepos = - cfg.getBoolean("remote", rc.getName(), "createMissingRepositories", true); - replicatePermissions = - cfg.getBoolean("remote", rc.getName(), "replicatePermissions", true); - replicateProjectDeletions = - cfg.getBoolean("remote", rc.getName(), "replicateProjectDeletions", false); - remoteNameStyle = MoreObjects.firstNonNull( - cfg.getString("remote", rc.getName(), "remoteNameStyle"), "slash"); - projects = cfg.getStringList("remote", rc.getName(), "projects"); - final CurrentUser remoteUser; - authGroupNames = cfg.getStringList("remote", rc.getName(), "authGroup"); - if (authGroupNames.length > 0) { + if (!cfg.getAuthGroupNames().isEmpty()) { ImmutableSet.Builder<AccountGroup.UUID> builder = ImmutableSet.builder(); - for (String name : authGroupNames) { + for (String name : cfg.getAuthGroupNames()) { GroupReference g = GroupBackends.findExactSuggestion(groupBackend, name); if (g != null) { builder.add(g.getUUID()); @@ -159,7 +131,7 @@ class Destination { bind(PerRequestProjectControlCache.class).in(RequestScoped.class); bind(Destination.class).toInstance(Destination.this); - bind(RemoteConfig.class).toInstance(remote); + bind(RemoteConfig.class).toInstance(config.getRemoteConfig()); install(new FactoryModuleBuilder().build(PushOne.Factory.class)); } @@ -203,11 +175,18 @@ class Destination { } } - void start(WorkQueue workQueue) { - pool = workQueue.createQueue(poolThreads, poolName); + public QueueInfo getQueueInfo() { + synchronized (stateLock) { + return new QueueInfo(pending, inFlight); + } + } + + public void start(WorkQueue workQueue) { + String poolName = "ReplicateTo-" + config.getRemoteConfig().getName(); + pool = workQueue.createQueue(config.getPoolThreads(), poolName); } - int shutdown() { + public int shutdown() { int cnt = 0; if (pool != null) { for (Runnable r : pool.getQueue()) { @@ -220,43 +199,35 @@ class Destination { return cnt; } - private static int getInt( - RemoteConfig rc, Config cfg, String name, int defValue) { - return cfg.getInt("remote", rc.getName(), name, defValue); - } - - private static int getTimeUnit( - RemoteConfig rc, Config cfg, String name, int defValue, TimeUnit unit) { - return (int)ConfigUtil.getTimeUnit( - cfg, "remote", rc.getName(), name, defValue, unit); - } - - private boolean isVisible(final Project.NameKey project, + private boolean shouldReplicate(final Project.NameKey project, ReplicationState... states) { try { return threadScoper.scope(new Callable<Boolean>() { @Override public Boolean call() throws NoSuchProjectException { - return controlFor(project).isVisible(); + ProjectControl projectControl = controlFor(project); + return projectControl.isReadable() && (!projectControl.isHidden() + || config.replicateHiddenProjects()); } }).call(); } catch (NoSuchProjectException err) { stateLog.error(String.format("source project %s not available", project), err, states); } catch (Exception e) { - throw Throwables.propagate(e); + Throwables.propagateIfPossible(e); + throw new RuntimeException(e); } return false; } - void schedule(final Project.NameKey project, final String ref, - final URIish uri, ReplicationState state) { + void schedule(Project.NameKey project, String ref, URIish uri, + ReplicationState state) { repLog.info("scheduling replication {}:{} => {}", project, ref, uri); - if (!isVisible(project, state)) { + if (!shouldReplicate(project, state)) { return; } - if (!replicatePermissions) { + if (!config.replicatePermissions()) { PushOne e; synchronized (stateLock) { e = pending.get(uri); @@ -264,7 +235,7 @@ class Destination { if (e == null) { try (Repository git = gitManager.openRepository(project)) { try { - Ref head = git.getRef(Constants.HEAD); + Ref head = git.exactRef(Constants.HEAD); if (head != null && head.isSymbolic() && RefNames.REFS_CONFIG.equals(head.getLeaf().getName())) { @@ -287,14 +258,14 @@ class Destination { PushOne e = pending.get(uri); if (e == null) { e = opFactory.create(project, uri); - pool.schedule(e, delay, TimeUnit.SECONDS); + pool.schedule(e, config.getDelay(), TimeUnit.SECONDS); pending.put(uri, e); } e.addRef(ref); state.increasePushTaskCount(project.get(), ref); e.addState(ref, state); repLog.info("scheduled {}:{} => {} to run after {}s", project, ref, - e, delay); + e, config.getDelay()); } } @@ -377,12 +348,13 @@ class Destination { pending.put(uri, pushOp); switch (reason) { case COLLISION: - pool.schedule(pushOp, delay, TimeUnit.SECONDS); + pool.schedule(pushOp, config.getDelay(), TimeUnit.SECONDS); break; case TRANSPORT_ERROR: + case REPOSITORY_MISSING: default: pushOp.setToRetry(); - pool.schedule(pushOp, retryDelay, TimeUnit.MINUTES); + pool.schedule(pushOp, config.getRetryDelay(), TimeUnit.MINUTES); break; } } @@ -414,23 +386,25 @@ class Destination { } } - boolean wouldPushProject(final Project.NameKey project) { - if (!isVisible(project)) { + boolean wouldPushProject(Project.NameKey project) { + if (!shouldReplicate(project)) { return false; } // by default push all projects - if (projects.length < 1) { + List<String> projects = config.getProjects(); + if (projects.isEmpty()) { return true; } - return (new ReplicationFilter(Arrays.asList(projects))).matches(project); + return (new ReplicationFilter(projects)).matches(project); } boolean isSingleProjectMatch() { - boolean ret = (projects.length == 1); + List<String> projects = config.getProjects(); + boolean ret = (projects.size() == 1); if (ret) { - String projectMatch = projects[0]; + String projectMatch = projects.get(0); if (ReplicationFilter.getPatternType(projectMatch) != ReplicationFilter.PatternType.EXACT_MATCH) { // projectMatch is either regular expression, or wild-card. @@ -445,10 +419,10 @@ class Destination { } boolean wouldPushRef(String ref) { - if (!replicatePermissions && RefNames.REFS_CONFIG.equals(ref)) { + if (!config.replicatePermissions() && RefNames.REFS_CONFIG.equals(ref)) { return false; } - for (RefSpec s : remote.getPushRefSpecs()) { + for (RefSpec s : config.getRemoteConfig().getPushRefSpecs()) { if (s.matchSource(ref)) { return true; } @@ -457,28 +431,30 @@ class Destination { } boolean isCreateMissingRepos() { - return createMissingRepos; + return config.createMissingRepos(); } boolean isReplicatePermissions() { - return replicatePermissions; + return config.replicatePermissions(); } boolean isReplicateProjectDeletions() { - return replicateProjectDeletions; + return config.replicateProjectDeletions(); } List<URIish> getURIs(Project.NameKey project, String urlMatch) { - List<URIish> r = Lists.newArrayListWithCapacity(remote.getURIs().size()); - for (URIish uri : remote.getURIs()) { + List<URIish> r = Lists.newArrayListWithCapacity( + config.getRemoteConfig().getURIs().size()); + for (URIish uri : config.getRemoteConfig().getURIs()) { if (matches(uri, urlMatch)) { String name = project.get(); if (needsUrlEncoding(uri)) { name = encode(name); } + String remoteNameStyle = config.getRemoteNameStyle(); if (remoteNameStyle.equals("dash")) { name = name.replace("/", "-"); - } else if(remoteNameStyle.equals("underscore")) { + } else if (remoteNameStyle.equals("underscore")) { name = name.replace("/", "_"); } else if (remoteNameStyle.equals("basenameOnly")) { name = FilenameUtils.getBaseName(name); @@ -519,28 +495,28 @@ class Destination { } } - String[] getAdminUrls() { - return adminUrls; + ImmutableList<String> getAdminUrls() { + return config.getAdminUrls(); } - String[] getUrls() { - return urls; + ImmutableList<String> getUrls() { + return config.getUrls(); } - RemoteConfig getRemoteConfig() { - return remote; + ImmutableList<String> getAuthGroupNames() { + return config.getAuthGroupNames(); } - String[] getAuthGroupNames() { - return authGroupNames; + ImmutableList<String> getProjects() { + return config.getProjects(); } - String[] getProjects() { - return projects; + int getLockErrorMaxRetries() { + return config.getLockErrorMaxRetries(); } - int getLockErrorMaxRetries() { - return lockErrorMaxRetries; + String getRemoteConfigName() { + return config.getRemoteConfig().getName(); } private static boolean matches(URIish uri, String urlMatch) { diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java new file mode 100644 index 0000000..0d7d3ce --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java @@ -0,0 +1,127 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; + +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.transport.RemoteConfig; + +class DestinationConfiguration { + private final int delay; + private final int retryDelay; + private final int lockErrorMaxRetries; + private final ImmutableList<String> adminUrls; + private final int poolThreads; + private final boolean createMissingRepos; + private final boolean replicatePermissions; + private final boolean replicateProjectDeletions; + private final boolean replicateHiddenProjects; + private final String remoteNameStyle; + private final ImmutableList<String> urls; + private final ImmutableList<String> projects; + private final ImmutableList<String> authGroupNames; + private final RemoteConfig remoteConfig; + + DestinationConfiguration(RemoteConfig remoteConfig, Config cfg) { + this.remoteConfig = remoteConfig; + String name = remoteConfig.getName(); + urls = ImmutableList.copyOf( + cfg.getStringList("remote", name, "url")); + delay = Math.max(0, getInt(remoteConfig, cfg, "replicationdelay", 15)); + projects = ImmutableList.copyOf( + cfg.getStringList("remote", name, "projects")); + adminUrls = ImmutableList.copyOf( + cfg.getStringList("remote", name, "adminUrl")); + retryDelay = Math.max(0, getInt(remoteConfig, cfg, "replicationretry", 1)); + poolThreads = Math.max(0, getInt(remoteConfig, cfg, "threads", 1)); + authGroupNames = ImmutableList.copyOf( + cfg.getStringList("remote", name, "authGroup")); + lockErrorMaxRetries = cfg.getInt("replication", "lockErrorMaxRetries", 0); + + createMissingRepos = + cfg.getBoolean("remote", name, "createMissingRepositories", true); + replicatePermissions = + cfg.getBoolean("remote", name, "replicatePermissions", true); + replicateProjectDeletions = + cfg.getBoolean("remote", name, "replicateProjectDeletions", false); + replicateHiddenProjects = + cfg.getBoolean("remote", name, "replicateHiddenProjects", false); + remoteNameStyle = MoreObjects.firstNonNull( + cfg.getString("remote", name, "remoteNameStyle"), "slash"); + } + + public int getDelay() { + return delay; + } + + public int getRetryDelay() { + return retryDelay; + } + + public int getPoolThreads() { + return poolThreads; + } + + public int getLockErrorMaxRetries() { + return lockErrorMaxRetries; + } + + public ImmutableList<String> getUrls() { + return urls; + } + + public ImmutableList<String> getAdminUrls() { + return adminUrls; + } + + public ImmutableList<String> getProjects() { + return projects; + } + + public ImmutableList<String> getAuthGroupNames() { + return authGroupNames; + } + + public String getRemoteNameStyle() { + return remoteNameStyle; + } + + public boolean replicatePermissions() { + return replicatePermissions; + } + + public boolean createMissingRepos() { + return createMissingRepos; + } + + public boolean replicateProjectDeletions() { + return replicateProjectDeletions; + } + + public boolean replicateHiddenProjects() { + return replicateHiddenProjects; + } + + public RemoteConfig getRemoteConfig() { + return remoteConfig; + } + + private static int getInt( + RemoteConfig rc, Config cfg, String name, int defValue) { + return cfg.getInt("remote", rc.getName(), name, defValue); + } +} diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ListCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ListCommand.java index 91fa65a..247ebf7 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ListCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ListCommand.java @@ -28,15 +28,16 @@ import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType; import org.kohsuke.args4j.Option; +import java.util.Collection; import java.util.List; @RequiresCapability(GlobalCapability.ADMINISTRATE_SERVER) -@CommandMetaData(name = "list", description = "List specific remote destinations information") +@CommandMetaData(name = "list", description = "List remote destination information") final class ListCommand extends SshCommand { @Option(name = "--remote", metaVar = "PATTERN", usage = "pattern to match remote name on") private String remote; - @Option(name = "--detail", usage = "print remote destination detail information") + @Option(name = "--detail", usage = "output detailed information") private boolean detail; @Option(name = "--json", usage = "output in json format") @@ -47,10 +48,9 @@ final class ListCommand extends SshCommand { @Override protected void run() { - List<Destination> dest = config.getDestinations(FilterType.ALL); - for (Destination d : dest) { - if (matches(d.getRemoteConfig().getName())) { - printRemote(d, detail); + for (Destination d : config.getDestinations(FilterType.ALL)) { + if (matches(d.getRemoteConfigName())) { + printRemote(d); } } } @@ -61,8 +61,8 @@ final class ListCommand extends SshCommand { || name.matches(remote)); } - private void addProperty(JsonObject obj, String key, String[] values) { - if (values.length > 0) { + private void addProperty(JsonObject obj, String key, List<String> values) { + if (!values.isEmpty()) { JsonArray list = new JsonArray(); for (String v : values) { list.add(new JsonPrimitive(v)); @@ -71,21 +71,43 @@ final class ListCommand extends SshCommand { } } - private void printRemote(Destination d, boolean detail) { + private void addQueueDetails(StringBuilder out, Collection<PushOne> values) { + for (PushOne p : values) { + out.append(" ") + .append(p.toString()) + .append("\n"); + } + } + + private void addQueueDetails(JsonObject obj, String key, + Collection<PushOne> values) { + if (values.size() > 0) { + JsonArray list = new JsonArray(); + for (PushOne p : values) { + list.add(new JsonPrimitive(p.toString())); + } + obj.add(key, list); + } + } + + private void printRemote(Destination d) { if (json) { JsonObject obj = new JsonObject(); - obj.addProperty("remote", d.getRemoteConfig().getName()); + obj.addProperty("Remote", d.getRemoteConfigName()); addProperty(obj, "Url", d.getUrls()); if (detail) { addProperty(obj, "AdminUrl", d.getAdminUrls()); addProperty(obj, "AuthGroup", d.getAuthGroupNames()); addProperty(obj, "Project", d.getProjects()); + Destination.QueueInfo q = d.getQueueInfo(); + addQueueDetails(obj, "InFlight", q.inFlight.values()); + addQueueDetails(obj, "Pending", q.pending.values()); } stdout.print(obj.toString() + "\n"); } else { StringBuilder out = new StringBuilder(); out.append("Remote: ") - .append(d.getRemoteConfig().getName()) + .append(d.getRemoteConfigName()) .append("\n"); for (String url : d.getUrls()) { out.append("Url: ") @@ -111,6 +133,16 @@ final class ListCommand extends SshCommand { .append(project) .append("\n"); } + + Destination.QueueInfo q = d.getQueueInfo(); + out.append("In Flight: ") + .append(q.inFlight.size()) + .append("\n"); + addQueueDetails(out, q.inFlight.values()); + out.append("Pending: ") + .append(q.pending.size()) + .append("\n"); + addQueueDetails(out, q.pending.values()); } stdout.print(out.toString() + "\n"); } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java b/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java index a5df0a7..5accf95 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java @@ -17,6 +17,7 @@ package com.googlesource.gerrit.plugins.replication; import com.google.common.util.concurrent.Atomics; import com.google.gerrit.common.EventDispatcher; import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.extensions.systemstatus.ServerInformation; import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gwtorm.server.SchemaFactory; @@ -28,22 +29,22 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -class OnStartStop implements LifecycleListener { +public class OnStartStop implements LifecycleListener { private final AtomicReference<Future<?>> pushAllFuture; private final ServerInformation srvInfo; private final PushAll.Factory pushAll; private final ReplicationQueue queue; private final ReplicationConfig config; + private final DynamicItem<EventDispatcher> eventDispatcher; private final SchemaFactory<ReviewDb> database; - private final EventDispatcher eventDispatcher; @Inject - OnStartStop( + protected OnStartStop( ServerInformation srvInfo, PushAll.Factory pushAll, ReplicationQueue queue, ReplicationConfig config, - EventDispatcher eventDispatcher, + DynamicItem<EventDispatcher> eventDispatcher, SchemaFactory<ReviewDb> database) { this.srvInfo = srvInfo; this.pushAll = pushAll; @@ -60,9 +61,8 @@ class OnStartStop implements LifecycleListener { if (srvInfo.getState() == ServerInformation.State.STARTUP && config.isReplicateAllOnPluginStart()) { - ReplicationState state = - new ReplicationState(new GitUpdateProcessing(eventDispatcher, - database)); + ReplicationState state = new ReplicationState( + new GitUpdateProcessing(eventDispatcher.get(), database)); pushAllFuture.set(pushAll.create( null, ReplicationFilter.all(), state).schedule(30, TimeUnit.SECONDS)); } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java index 5ff4035..16e1678 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java @@ -24,10 +24,10 @@ import com.google.inject.assistedinject.Assisted; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -class PushAll implements Runnable { +public class PushAll implements Runnable { private final ReplicationStateListener stateLog; - interface Factory { + public interface Factory { PushAll create(String urlMatch, ReplicationFilter filter, ReplicationState state); @@ -41,7 +41,7 @@ class PushAll implements Runnable { private final ReplicationState state; @Inject - PushAll(WorkQueue wq, + protected PushAll(WorkQueue wq, ProjectCache projectCache, ReplicationQueue rq, ReplicationStateListener stateLog, diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java index 66304bc..0c74a6f 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java @@ -15,15 +15,14 @@ package com.googlesource.gerrit.plugins.replication; import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; +import static java.util.concurrent.TimeUnit.NANOSECONDS; import com.google.common.base.Throwables; import com.google.common.collect.LinkedListMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import com.google.gerrit.common.TimeUtil; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; +import com.google.gerrit.metrics.Timer1; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.client.RefNames; import com.google.gerrit.reviewdb.server.ReviewDb; @@ -64,8 +63,10 @@ import org.eclipse.jgit.transport.URIish; import org.slf4j.MDC; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -111,21 +112,23 @@ class PushOne implements ProjectRunnable { private int lockRetryCount; private final int id; private final long createdAt; + private final ReplicationMetrics metrics; @Inject - PushOne(final GitRepositoryManager grm, - final SchemaFactory<ReviewDb> s, - final Destination p, - final RemoteConfig c, - final CredentialsFactory cpFactory, - final TagCache tc, - final PerThreadRequestScope.Scoper ts, - final ChangeCache cc, - final ReplicationQueue rq, - final IdGenerator ig, - final ReplicationStateListener sl, - @Assisted final Project.NameKey d, - @Assisted final URIish u) { + PushOne(GitRepositoryManager grm, + SchemaFactory<ReviewDb> s, + Destination p, + RemoteConfig c, + CredentialsFactory cpFactory, + TagCache tc, + PerThreadRequestScope.Scoper ts, + ChangeCache cc, + ReplicationQueue rq, + IdGenerator ig, + ReplicationStateListener sl, + ReplicationMetrics m, + @Assisted Project.NameKey d, + @Assisted URIish u) { gitManager = grm; schema = s; pool = p; @@ -141,7 +144,8 @@ class PushOne implements ProjectRunnable { maxLockRetries = pool.getLockErrorMaxRetries(); id = ig.next(); stateLog = sl; - createdAt = TimeUtil.nowMs(); + createdAt = System.nanoTime(); + metrics = m; } @Override @@ -252,7 +256,7 @@ class PushOne implements ProjectRunnable { @Override public void run() { try { - threadScoper.scope(new Callable<Void>(){ + threadScoper.scope(new Callable<Void>() { @Override public Void call() { runPushOperation(); @@ -260,7 +264,8 @@ class PushOne implements ProjectRunnable { } }).call(); } catch (Exception e) { - throw Throwables.propagate(e); + Throwables.propagateIfPossible(e); + throw new RuntimeException(e); } finally { statesCleanUp(); } @@ -281,15 +286,18 @@ class PushOne implements ProjectRunnable { return; } - long startedAt = TimeUtil.nowMs(); repLog.info("Replication to " + uri + " started..."); + Timer1.Context context = metrics.start(config.getName()); try { + long startedAt = context.getStartTime(); + long delay = NANOSECONDS.toMillis(startedAt - createdAt); + metrics.record(config.getName(), delay, retryCount); git = gitManager.openRepository(projectName); runImpl(); - long finishedAt = TimeUtil.nowMs(); + long elapsed = NANOSECONDS.toMillis(context.stop()); repLog.info("Replication to " + uri + " completed in " - + (finishedAt - startedAt) + "ms, " - + (startedAt - createdAt) + "ms delay, " + retryCount + " retries"); + + (elapsed) + "ms, " + + (delay) + "ms delay, " + retryCount + " retries"); } catch (RepositoryNotFoundException e) { stateLog.error("Cannot replicate " + projectName + "; Local repository error: " @@ -299,7 +307,7 @@ class PushOne implements ProjectRunnable { // Tried to replicate to a remote via anonymous git:// but the repository // does not exist. In this case NoRemoteRepositoryException is not // raised. - final String msg = e.getMessage(); + String msg = e.getMessage(); if (msg.contains("access denied") || msg.contains("no such repository")) { createRepository(); } else { @@ -350,7 +358,7 @@ class PushOne implements ProjectRunnable { private void createRepository() { if (pool.isCreateMissingRepos()) { try { - final Ref head = git.getRef(Constants.HEAD); + Ref head = git.exactRef(Constants.HEAD); if (replicationQueue.createProject(projectName, head != null ? head.getName() : null)) { repLog.warn("Missing repository created; retry replication to " + uri); pool.reschedule(this, Destination.RetryReason.REPOSITORY_MISSING); @@ -369,18 +377,10 @@ class PushOne implements ProjectRunnable { } private void runImpl() throws IOException { - Transport tn = Transport.open(git, uri); PushResult res; - try { + try (Transport tn = Transport.open(git, uri)) { res = pushVia(tn); - } finally { - try { - tn.close(); - } catch (Throwable e2) { - repLog.warn("Unexpected error while closing " + uri, e2); - } } - updateStates(res.getRemoteUpdates()); } @@ -418,7 +418,7 @@ class PushOne implements ProjectRunnable { // If we aren't mirroring, reduce the space we need to filter // to only the references we will update during this operation. // - Map<String, Ref> n = Maps.newHashMap(); + Map<String, Ref> n = new HashMap<>(); for (String src : delta) { Ref r = local.get(src); if (r != null) { @@ -442,7 +442,7 @@ class PushOne implements ProjectRunnable { private List<RemoteRefUpdate> doPushAll(Transport tn, Map<String, Ref> local) throws NotSupportedException, TransportException, IOException { - List<RemoteRefUpdate> cmds = Lists.newArrayList(); + List<RemoteRefUpdate> cmds = new ArrayList<>(); boolean noPerms = !pool.isReplicatePermissions(); Map<String, Ref> remote = listRemote(tn); for (Ref src : local.values()) { @@ -476,7 +476,7 @@ class PushOne implements ProjectRunnable { private List<RemoteRefUpdate> doPushDelta(Map<String, Ref> local) throws IOException { - List<RemoteRefUpdate> cmds = Lists.newArrayList(); + List<RemoteRefUpdate> cmds = new ArrayList<>(); boolean noPerms = !pool.isReplicatePermissions(); for (String src : delta) { RefSpec spec = matchSrc(src); @@ -506,11 +506,8 @@ class PushOne implements ProjectRunnable { private Map<String, Ref> listRemote(Transport tn) throws NotSupportedException, TransportException { - FetchConnection fc = tn.openFetch(); - try { + try (FetchConnection fc = tn.openFetch()) { return fc.getRefsMap(); - } finally { - fc.close(); } } @@ -621,7 +618,7 @@ class PushOne implements ProjectRunnable { public static class LockFailureException extends TransportException { private static final long serialVersionUID = 1L; - public LockFailureException(URIish uri, String message) { + LockFailureException(URIish uri, String message) { super(uri, message); } } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java index c63d346..4a84d89 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java @@ -15,10 +15,6 @@ package com.googlesource.gerrit.plugins.replication; import com.google.gerrit.common.EventDispatcher; -import com.google.gerrit.reviewdb.client.Branch; -import com.google.gerrit.reviewdb.client.Change; -import com.google.gerrit.reviewdb.client.PatchSet; -import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.server.events.RefEvent; import com.google.gwtorm.server.OrmException; @@ -173,42 +169,25 @@ public abstract class PushResultProcessing { @Override void onRefReplicatedToOneNode(String project, String ref, URIish uri, RefPushResult status, RemoteRefUpdate.Status refStatus) { - RefReplicatedEvent event = - new RefReplicatedEvent(project, ref, resolveNodeName(uri), status, refStatus); - postEvent(project, ref, event); + postEvent(new RefReplicatedEvent(project, ref, resolveNodeName(uri), + status, refStatus)); } @Override void onRefReplicatedToAllNodes(String project, String ref, int nodesCount) { - RefReplicationDoneEvent event = - new RefReplicationDoneEvent(project, ref, nodesCount); - postEvent(project, ref, event); + postEvent(new RefReplicationDoneEvent(project, ref, nodesCount)); } @Override void onAllRefsReplicatedToAllNodes(int totalPushTasksCount) { } - private void postEvent(String project, String ref, RefEvent event) { - if (PatchSet.isChangeRef(ref)) { - try (ReviewDb db = schema.open()) { - Change change = retrieveChange(ref, db); - if (change != null) { - dispatcher.postEvent(change, event, db); - } - } catch (Exception e) { - log.error("Cannot post event", e); - } - } else { - Branch.NameKey branch = new Branch.NameKey(Project.NameKey.parse(project), ref); - dispatcher.postEvent(branch, event); + private void postEvent(RefEvent event) { + try (ReviewDb db = schema.open()) { + dispatcher.postEvent(event, db); + } catch (OrmException e) { + log.error("Cannot post event", e); } } - - private Change retrieveChange(String ref, ReviewDb db) throws OrmException { - PatchSet.Id id = PatchSet.Id.fromRef(ref); - Change change = db.changes().get(id.getParentKey()); - return change; - } } } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEvent.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEvent.java index f200194..a1c5596 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEvent.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEvent.java @@ -23,26 +23,24 @@ import org.eclipse.jgit.transport.RemoteRefUpdate; import org.eclipse.jgit.transport.RemoteRefUpdate.Status; public class RefReplicatedEvent extends RefEvent { - public final String project; - public final String ref; - public final String targetNode; - public final String status; - public final Status refStatus; + static final String TYPE = "ref-replicated"; + + final String project; + final String ref; + final String targetNode; + final String status; + final Status refStatus; public RefReplicatedEvent(String project, String ref, String targetNode, RefPushResult status, RemoteRefUpdate.Status refStatus) { - super("ref-replicated"); + super(TYPE); this.project = project; this.ref = ref; this.targetNode = targetNode; - this.status = toStatusString(status); + this.status = status.toString(); this.refStatus = refStatus; } - private String toStatusString(RefPushResult status) { - return status.name().toLowerCase().replace("_", "-"); - } - @Override public Project.NameKey getProjectNameKey() { return new Project.NameKey(project); diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEvent.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEvent.java index fe92bc8..90595b3 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEvent.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEvent.java @@ -18,12 +18,14 @@ import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.events.RefEvent; public class RefReplicationDoneEvent extends RefEvent { - public final String project; - public final String ref; - public final int nodesCount; + static final String TYPE = "ref-replication-done"; + + final String project; + final String ref; + final int nodesCount; public RefReplicationDoneEvent(String project, String ref, int nodesCount) { - super("ref-replication-done"); + super(TYPE); this.project = project; this.ref = ref; this.nodesCount = nodesCount; diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java index f9aa668..9d8c3ca 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java @@ -14,7 +14,6 @@ package com.googlesource.gerrit.plugins.replication; -import com.google.gerrit.reviewdb.client.AccountProjectWatch; import com.google.gerrit.reviewdb.client.Change; import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.account.CapabilityControl; @@ -22,12 +21,11 @@ import com.google.gerrit.server.account.GroupMembership; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; -import java.util.Collection; import java.util.Collections; import java.util.Set; -class RemoteSiteUser extends CurrentUser { - interface Factory { +public class RemoteSiteUser extends CurrentUser { + public interface Factory { RemoteSiteUser create(@Assisted GroupMembership authGroups); } @@ -49,9 +47,4 @@ class RemoteSiteUser extends CurrentUser { public Set<Change.Id> getStarredChanges() { return Collections.emptySet(); } - - @Override - public Collection<AccountProjectWatch> getNotificationFilters() { - return Collections.emptySet(); - } } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java index f7dc733..4b976bc 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java @@ -60,12 +60,15 @@ public class ReplicationFileBasedConfig implements ReplicationConfig { private final GroupIncludeCache groupIncludeCache; @Inject - public ReplicationFileBasedConfig(final Injector injector, final SitePaths site, - final RemoteSiteUser.Factory ruf, final PluginUser pu, - final GitRepositoryManager grm, - final GroupBackend gb, - final ReplicationStateListener stateLog, - final GroupIncludeCache groupIncludeCache) throws ConfigInvalidException, IOException { + public ReplicationFileBasedConfig(Injector injector, + SitePaths site, + RemoteSiteUser.Factory ruf, + PluginUser pu, + GitRepositoryManager grm, + GroupBackend gb, + ReplicationStateListener stateLog, + GroupIncludeCache groupIncludeCache) + throws ConfigInvalidException, IOException { this.cfgPath = site.etc_dir.resolve("replication.config"); this.groupIncludeCache = groupIncludeCache; this.injector = injector; @@ -165,8 +168,9 @@ public class ReplicationFileBasedConfig implements ReplicationConfig { } Destination destination = - new Destination(injector, c, config, replicationUserFactory, - pluginUser, gitRepositoryManager, groupBackend, stateLog, groupIncludeCache); + new Destination(injector, new DestinationConfiguration(c, + config), replicationUserFactory, pluginUser, + gitRepositoryManager, groupBackend, stateLog, groupIncludeCache); if (!destination.isSingleProjectMatch()) { for (URIish u : c.getURIs()) { diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationMetrics.java new file mode 100644 index 0000000..e02084d --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationMetrics.java @@ -0,0 +1,77 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import com.google.gerrit.metrics.Description; +import com.google.gerrit.metrics.Field; +import com.google.gerrit.metrics.Histogram1; +import com.google.gerrit.metrics.MetricMaker; +import com.google.gerrit.metrics.Timer1; +import com.google.inject.Inject; +import com.google.inject.Singleton; + +@Singleton +public class ReplicationMetrics { + Timer1<String> executionTime; + Histogram1<String> executionDelay; + Histogram1<String> executionRetries; + + @Inject + ReplicationMetrics(MetricMaker metricMaker) { + Field<String> DEST_FIELD = Field.ofString("destination"); + + executionTime = metricMaker.newTimer( + "replication_latency", + new Description("Time spent pushing to remote destination.") + .setCumulative() + .setUnit(Description.Units.MILLISECONDS), + DEST_FIELD); + + executionDelay = metricMaker.newHistogram( + "replication_delay", + new Description("Time spent waiting before pushing to remote destination") + .setCumulative() + .setUnit(Description.Units.MILLISECONDS), + DEST_FIELD); + + executionRetries = metricMaker.newHistogram( + "replication_retries", + new Description("Number of retries when pushing to remote destination") + .setCumulative() + .setUnit("retries"), + DEST_FIELD); + } + + /** + * Start the replication latency timer for a destination. + * @param name the destination name. + * @return the timer context. + */ + Timer1.Context start(String name) { + return executionTime.start(name); + } + + /** + * Record the replication delay and retry metrics for a destination. + * @param name the destination name. + * @param delay replication delay in milliseconds. + * @param retries number of retries. + */ + void record(String name, long delay, long retries) { + executionDelay.record(name, delay); + executionRetries.record(name, retries); + } + +} diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java index a5d0b82..5a5f3b4 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java @@ -14,7 +14,6 @@ package com.googlesource.gerrit.plugins.replication; -import static com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult.SUCCEEDED; import static com.googlesource.gerrit.plugins.replication.StartReplicationCapability.START_REPLICATION; import com.google.gerrit.extensions.annotations.Exports; @@ -31,8 +30,7 @@ import com.google.inject.Scopes; import com.google.inject.assistedinject.FactoryModuleBuilder; import com.google.inject.internal.UniqueAnnotations; -import org.eclipse.jgit.transport.RemoteRefUpdate; - +import org.eclipse.jgit.transport.SshSessionFactory; class ReplicationModule extends AbstractModule { @Override @@ -66,8 +64,9 @@ class ReplicationModule extends AbstractModule { bind(ReplicationConfig.class).to(AutoReloadConfigDecorator.class); bind(ReplicationStateListener.class).to(ReplicationStateLogger.class); - EventTypes.registerClass(new RefReplicatedEvent(null, null, null, - SUCCEEDED, RemoteRefUpdate.Status.OK)); - EventTypes.registerClass(new RefReplicationDoneEvent(null, null, 0)); + EventTypes.register(RefReplicatedEvent.TYPE, RefReplicatedEvent.class); + EventTypes.register(RefReplicationDoneEvent.TYPE, RefReplicationDoneEvent.class); + bind(SshSessionFactory.class).toProvider( + ReplicationSshSessionFactoryProvider.class); } } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java index 67d172f..fc11773 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java @@ -15,18 +15,19 @@ package com.googlesource.gerrit.plugins.replication; import com.google.common.base.Strings; -import com.google.common.collect.Sets; import com.google.gerrit.common.EventDispatcher; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; import com.google.gerrit.extensions.events.HeadUpdatedListener; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.extensions.events.NewProjectCreatedListener; import com.google.gerrit.extensions.events.ProjectDeletedListener; +import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.server.git.WorkQueue; import com.google.gwtorm.server.SchemaFactory; import com.google.inject.Inject; +import com.google.inject.Provider; import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing; import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType; @@ -50,11 +51,11 @@ import java.io.IOException; import java.io.OutputStream; import java.net.URISyntaxException; import java.util.Collections; -import java.util.List; +import java.util.HashSet; import java.util.Set; /** Manages automatic replication to remote repositories. */ -class ReplicationQueue implements +public class ReplicationQueue implements LifecycleListener, GitReferenceUpdatedListener, NewProjectCreatedListener, @@ -80,21 +81,24 @@ class ReplicationQueue implements private final WorkQueue workQueue; private final SchemaFactory<ReviewDb> database; - private final EventDispatcher dispatcher; + private final DynamicItem<EventDispatcher> dispatcher; private final ReplicationConfig config; + private final Provider<SshSessionFactory> sshSessionFactoryProvider; private volatile boolean running; @Inject - ReplicationQueue(final WorkQueue wq, - final ReplicationConfig rc, - final SchemaFactory<ReviewDb> db, - final EventDispatcher dis, - final ReplicationStateListener sl) { + ReplicationQueue(WorkQueue wq, + ReplicationConfig rc, + SchemaFactory<ReviewDb> db, + DynamicItem<EventDispatcher> dis, + ReplicationStateListener sl, + Provider<SshSessionFactory> sshSessionFactoryProvider) { workQueue = wq; database = db; dispatcher = dis; config = rc; stateLog = sl; + this.sshSessionFactoryProvider = sshSessionFactoryProvider; } @Override @@ -132,7 +136,8 @@ class ReplicationQueue implements @Override public void onGitReferenceUpdated(GitReferenceUpdatedListener.Event event) { - ReplicationState state = new ReplicationState(new GitUpdateProcessing(dispatcher, database)); + ReplicationState state = + new ReplicationState(new GitUpdateProcessing(dispatcher.get(), database)); if (!running) { stateLog.warn("Replication plugin did not finish startup before event", state); return; @@ -183,17 +188,15 @@ class ReplicationQueue implements return Collections.emptySet(); } - Set<URIish> uris = Sets.newHashSet(); + Set<URIish> uris = new HashSet<>(); for (Destination config : this.config.getDestinations(filterType)) { if (!config.wouldPushProject(projectName)) { continue; } - List<URIish> uriList = config.getURIs(projectName, "*"); - String[] adminUrls = config.getAdminUrls(); boolean adminURLUsed = false; - for (String url : adminUrls) { + for (String url : config.getAdminUrls()) { if (Strings.isNullOrEmpty(url)) { continue; } @@ -227,7 +230,7 @@ class ReplicationQueue implements } if (!adminURLUsed) { - for (URIish uri : uriList) { + for (URIish uri : config.getURIs(projectName, "*")) { uris.add(uri); } } @@ -274,7 +277,7 @@ class ReplicationQueue implements } } - private static void createRemoteSsh(URIish uri, String head) { + private void createRemoteSsh(URIish uri, String head) { String quotedPath = QuotedString.BOURNE.quote(uri.getPath()); String cmd = "mkdir -p " + quotedPath + " && cd " + quotedPath @@ -319,7 +322,7 @@ class ReplicationQueue implements } } - public static void recursivelyDelete(File dir) throws IOException { + private static void recursivelyDelete(File dir) throws IOException { File[] contents = dir.listFiles(); if (contents != null) { for (File d : contents) { @@ -337,7 +340,7 @@ class ReplicationQueue implements } } - private static void deleteRemoteSsh(URIish uri) { + private void deleteRemoteSsh(URIish uri) { String quotedPath = QuotedString.BOURNE.quote(uri.getPath()); String cmd = "rm -rf " + quotedPath; OutputStream errStream = newErrorBufferStream(); @@ -366,7 +369,7 @@ class ReplicationQueue implements } } - private static void updateHeadRemoteSsh(URIish uri, String newHead) { + private void updateHeadRemoteSsh(URIish uri, String newHead) { String quotedPath = QuotedString.BOURNE.quote(uri.getPath()); String cmd = "cd " + quotedPath + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(newHead); @@ -396,7 +399,7 @@ class ReplicationQueue implements } } - private static void executeRemoteSsh(URIish uri, String cmd, + private void executeRemoteSsh(URIish uri, String cmd, OutputStream errStream) throws IOException { RemoteSession ssh = connect(uri); Process proc = ssh.exec(cmd, 0); @@ -417,8 +420,8 @@ class ReplicationQueue implements ssh.disconnect(); } - private static RemoteSession connect(URIish uri) throws TransportException { - return SshSessionFactory.getInstance().getSession(uri, null, FS.DETECTED, + private RemoteSession connect(URIish uri) throws TransportException { + return sshSessionFactoryProvider.get().getSession(uri, null, FS.DETECTED, SSH_REMOTE_TIMEOUT); } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationSshSessionFactoryProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationSshSessionFactoryProvider.java new file mode 100644 index 0000000..42bc284 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationSshSessionFactoryProvider.java @@ -0,0 +1,27 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import com.google.inject.Provider; + +import org.eclipse.jgit.transport.SshSessionFactory; + +class ReplicationSshSessionFactoryProvider implements Provider<SshSessionFactory> { + + @Override + public SshSessionFactory get() { + return SshSessionFactory.getInstance(); + } +} diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java index b29456d..27fe841 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java @@ -37,7 +37,7 @@ public class ReplicationState { private int nodesToReplicateCount; private int replicatedNodesCount; - public RefReplicationStatus(String project, String ref) { + RefReplicationStatus(String project, String ref) { this.project = project; this.ref = ref; } @@ -159,7 +159,7 @@ public class ReplicationState { public enum RefPushResult { /** - * The ref is not replicated to slave. + * The ref was not successfully replicated. */ FAILED, @@ -169,8 +169,13 @@ public class ReplicationState { NOT_ATTEMPTED, /** - * ref was successfully replicated. + * The ref was successfully replicated. */ SUCCEEDED; + + @Override + public String toString() { + return name().toLowerCase().replace("_", "-"); + } } } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateListener.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateListener.java index e5ac9d5..4dcf5a1 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateListener.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateListener.java @@ -28,7 +28,7 @@ public interface ReplicationStateListener { * @param msg message description of the error * @param states replication states impacted */ - public abstract void warn(String msg, ReplicationState... states); + void warn(String msg, ReplicationState... states); /** * Notify a fatal replication error. @@ -39,7 +39,7 @@ public interface ReplicationStateListener { * @param msg message description of the error * @param states replication states impacted */ - public abstract void error(String msg, ReplicationState... states); + void error(String msg, ReplicationState... states); /** * Notify a fatal replication error with the associated exception. @@ -50,7 +50,7 @@ public interface ReplicationStateListener { * @param t exception that caused the replication to fail * @param states replication states impacted */ - public abstract void error(String msg, Throwable t, + void error(String msg, Throwable t, ReplicationState... states); } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/StartReplicationCapability.java b/src/main/java/com/googlesource/gerrit/plugins/replication/StartReplicationCapability.java index 761be73..0cf0c2d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/StartReplicationCapability.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/StartReplicationCapability.java @@ -16,8 +16,8 @@ package com.googlesource.gerrit.plugins.replication; import com.google.gerrit.extensions.config.CapabilityDefinition; -class StartReplicationCapability extends CapabilityDefinition { - static final String START_REPLICATION = "startReplication"; +public class StartReplicationCapability extends CapabilityDefinition { + public static final String START_REPLICATION = "startReplication"; @Override public String getDescription() { diff --git a/src/main/resources/Documentation/cmd-list.md b/src/main/resources/Documentation/cmd-list.md index 84bb56b..3c6b78f 100644 --- a/src/main/resources/Documentation/cmd-list.md +++ b/src/main/resources/Documentation/cmd-list.md @@ -3,7 +3,7 @@ NAME ---- -@PLUGIN@ list - List specific remote destinations information +@PLUGIN@ list - List remote destination information. SYNOPSIS -------- @@ -16,8 +16,7 @@ ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ list DESCRIPTION ----------- -List all remote destinations information, or only those whose -name match the pattern given on the command line. +Lists the name and URL for remote destinations. ACCESS ------ @@ -25,19 +24,18 @@ Caller must be a member of the privileged 'Administrators' group. SCRIPTING --------- -This command is intended to be used in scripts. It is very useful -for replication status check for administrators as well. +This command is intended to be used in scripts. OPTIONS ------- `--remote <PATTERN>` -: Only print destinations whose remote name contains - the substring `PATTERN`. +: Only print information for destinations whose remote name matches + the `PATTERN`. `--detail` -: Print remote detail information: Name, Url, AdminUrl, - AuthGroup and Project. +: Print additional detailed information: AdminUrl, AuthGroup, Project + and queue (pending and in-flight). `--json` : Output in json format. diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index a528565..709d61f 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md @@ -262,6 +262,11 @@ remote.NAME.replicateProjectDeletions By default, false, do *not* replicate project deletions. +remote.NAME.replicateHiddenProjects +: If true, hidden projects will be replicated to the remote site. + + By default, false, do *not* replicate hidden projects. + remote.NAME.mirror : If true, replication will remove remote branches that are absent locally or invisible to the replication (for example read diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java index c6b2eb6..2395efe 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java @@ -17,7 +17,6 @@ package com.googlesource.gerrit.plugins.replication; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.replay; @@ -25,9 +24,6 @@ import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.verify; import com.google.gerrit.common.EventDispatcher; -import com.google.gerrit.reviewdb.client.Branch; -import com.google.gerrit.reviewdb.client.Change; -import com.google.gerrit.reviewdb.server.ChangeAccess; import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gwtorm.client.KeyUtil; import com.google.gwtorm.server.OrmException; @@ -51,7 +47,6 @@ public class GitUpdateProcessingTest extends TestCase { } private EventDispatcher dispatcherMock; - private ChangeAccess changeAccessMock; private GitUpdateProcessing gitUpdateProcessing; @Override @@ -59,10 +54,7 @@ public class GitUpdateProcessingTest extends TestCase { super.setUp(); dispatcherMock = createMock(EventDispatcher.class); replay(dispatcherMock); - changeAccessMock = createNiceMock(ChangeAccess.class); - replay(changeAccessMock); ReviewDb reviewDbMock = createNiceMock(ReviewDb.class); - expect(reviewDbMock.changes()).andReturn(changeAccessMock).anyTimes(); replay(reviewDbMock); SchemaFactory<ReviewDb> schemaMock = createMock(SchemaFactory.class); expect(schemaMock.open()).andReturn(reviewDbMock).anyTimes(); @@ -70,13 +62,13 @@ public class GitUpdateProcessingTest extends TestCase { gitUpdateProcessing = new GitUpdateProcessing(dispatcherMock, schemaMock); } - public void testHeadRefReplicated() throws URISyntaxException { + public void testHeadRefReplicated() throws URISyntaxException, OrmException { reset(dispatcherMock); RefReplicatedEvent expectedEvent = new RefReplicatedEvent("someProject", "refs/heads/master", "someHost", RefPushResult.SUCCEEDED, RemoteRefUpdate.Status.OK); - dispatcherMock.postEvent(anyObject(Branch.NameKey.class), - RefReplicatedEventEquals.eqEvent(expectedEvent)); + dispatcherMock.postEvent(RefReplicatedEventEquals.eqEvent(expectedEvent), + anyObject(ReviewDb.class)); expectLastCall().once(); replay(dispatcherMock); @@ -87,17 +79,11 @@ public class GitUpdateProcessingTest extends TestCase { } public void testChangeRefReplicated() throws URISyntaxException, OrmException { - Change expectedChange = new Change(null, null, null, null, null); - reset(changeAccessMock); - expect(changeAccessMock.get(anyObject(Change.Id.class))).andReturn(expectedChange); - replay(changeAccessMock); - reset(dispatcherMock); RefReplicatedEvent expectedEvent = new RefReplicatedEvent("someProject", "refs/changes/01/1/1", "someHost", RefPushResult.FAILED, RemoteRefUpdate.Status.REJECTED_NONFASTFORWARD); - dispatcherMock.postEvent(eq(expectedChange), - RefReplicatedEventEquals.eqEvent(expectedEvent), + dispatcherMock.postEvent(RefReplicatedEventEquals.eqEvent(expectedEvent), anyObject(ReviewDb.class)); expectLastCall().once(); replay(dispatcherMock); @@ -108,12 +94,13 @@ public class GitUpdateProcessingTest extends TestCase { verify(dispatcherMock); } - public void testOnAllNodesReplicated() { + public void testOnAllNodesReplicated() throws OrmException { reset(dispatcherMock); RefReplicationDoneEvent expectedDoneEvent = new RefReplicationDoneEvent("someProject", "refs/heads/master", 5); - dispatcherMock.postEvent(anyObject(Branch.NameKey.class), - RefReplicationDoneEventEquals.eqEvent(expectedDoneEvent)); + dispatcherMock.postEvent( + RefReplicationDoneEventEquals.eqEvent(expectedDoneEvent), + anyObject(ReviewDb.class)); expectLastCall().once(); replay(dispatcherMock); diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/PushReplicationTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/PushReplicationTest.java index 1e2df38..71b6600 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/PushReplicationTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/PushReplicationTest.java @@ -14,31 +14,34 @@ package com.googlesource.gerrit.plugins.replication; +import static com.google.common.truth.Truth.assertThat; import static com.googlesource.gerrit.plugins.replication.Destination.encode; import static com.googlesource.gerrit.plugins.replication.Destination.needsUrlEncoding; -import junit.framework.TestCase; - import org.eclipse.jgit.transport.URIish; +import org.junit.Test; import java.net.URISyntaxException; -public class PushReplicationTest extends TestCase { +public class PushReplicationTest { + + @Test public void testNeedsUrlEncoding() throws URISyntaxException { - assertTrue(needsUrlEncoding(new URIish("http://host/path"))); - assertTrue(needsUrlEncoding(new URIish("https://host/path"))); - assertTrue(needsUrlEncoding(new URIish("amazon-s3://config/bucket/path"))); - - assertFalse(needsUrlEncoding(new URIish("host:path"))); - assertFalse(needsUrlEncoding(new URIish("user@host:path"))); - assertFalse(needsUrlEncoding(new URIish("git://host/path"))); - assertFalse(needsUrlEncoding(new URIish("ssh://host/path"))); + assertThat(needsUrlEncoding(new URIish("http://host/path"))).isTrue(); + assertThat(needsUrlEncoding(new URIish("https://host/path"))).isTrue(); + assertThat(needsUrlEncoding(new URIish("amazon-s3://config/bucket/path"))).isTrue(); + + assertThat(needsUrlEncoding(new URIish("host:path"))).isFalse(); + assertThat(needsUrlEncoding(new URIish("user@host:path"))).isFalse(); + assertThat(needsUrlEncoding(new URIish("git://host/path"))).isFalse(); + assertThat(needsUrlEncoding(new URIish("ssh://host/path"))).isFalse(); } + @Test public void testUrlEncoding() { - assertEquals("foo/bar/thing", encode("foo/bar/thing")); - assertEquals("--%20All%20Projects%20--", encode("-- All Projects --")); - assertEquals("name/with%20a%20space", encode("name/with a space")); - assertEquals("name%0Awith-LF", encode("name\nwith-LF")); + assertThat(encode("foo/bar/thing")).isEqualTo("foo/bar/thing"); + assertThat(encode("-- All Projects --")).isEqualTo("--%20All%20Projects%20--"); + assertThat(encode("name/with a space")).isEqualTo("name/with%20a%20space"); + assertThat(encode("name\nwith-LF")).isEqualTo("name%0Awith-LF"); } } diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEventEquals.java b/src/test/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEventEquals.java index c68ba73..d614463 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEventEquals.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEventEquals.java @@ -48,6 +48,9 @@ public class RefReplicatedEventEquals implements IArgumentMatcher { if (!equals(expected.status, actualRefReplicatedEvent.status)) { return false; } + if (!equals(expected.refStatus, actualRefReplicatedEvent.refStatus)) { + return false; + } return true; } @@ -67,12 +70,14 @@ public class RefReplicatedEventEquals implements IArgumentMatcher { buffer.append(expected.getClass().getName()); buffer.append(" with project \""); buffer.append(expected.project); - buffer.append(" and ref \""); + buffer.append("\" and ref \""); buffer.append(expected.ref); - buffer.append(" and targetNode \""); + buffer.append("\" and targetNode \""); buffer.append(expected.targetNode); - buffer.append(" and status \""); + buffer.append("\" and status \""); buffer.append(expected.status); + buffer.append("\" and refStatus \""); + buffer.append(expected.refStatus); buffer.append("\")"); } } diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEventEquals.java b/src/test/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEventEquals.java index 42a25de..02f96fb 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEventEquals.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEventEquals.java @@ -52,7 +52,7 @@ public class RefReplicationDoneEventEquals implements IArgumentMatcher { if (object1 == object2) { return true; } - if (object1 != null && !object1.equals(object2)){ + if (object1 != null && !object1.equals(object2)) { return false; } return true; @@ -64,9 +64,9 @@ public class RefReplicationDoneEventEquals implements IArgumentMatcher { buffer.append(expected.getClass().getName()); buffer.append(" with project \""); buffer.append(expected.project); - buffer.append(" and ref \""); + buffer.append("\" and ref \""); buffer.append(expected.ref); - buffer.append(" and nodesCount \""); + buffer.append("\" and nodesCount \""); buffer.append(expected.nodesCount); buffer.append("\")"); } diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStateTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStateTest.java index 1408ed6..56096c2 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStateTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStateTest.java @@ -14,12 +14,12 @@ package com.googlesource.gerrit.plugins.replication; +import static com.google.common.truth.Truth.assertThat; import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.resetToDefault; import static org.easymock.EasyMock.verify; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult; @@ -44,13 +44,13 @@ public class ReplicationStateTest { @Test public void shouldNotHavePushTask() { - assertFalse(replicationState.hasPushTask()); + assertThat(replicationState.hasPushTask()).isFalse(); } @Test public void shouldHavePushTask() { replicationState.increasePushTaskCount("someProject", "someRef"); - assertTrue(replicationState.hasPushTask()); + assertThat(replicationState.hasPushTask()).isTrue(); } @Test @@ -217,4 +217,11 @@ public class ReplicationStateTest { replicationState.markAllPushTasksScheduled(); verify(pushResultProcessingMock); } + + @Test + public void toStringRefPushResult() throws Exception { + assertEquals("failed", RefPushResult.FAILED.toString()); + assertEquals("not-attempted", RefPushResult.NOT_ATTEMPTED.toString()); + assertEquals("succeeded", RefPushResult.SUCCEEDED.toString()); + } } |