diff options
31 files changed, 1540 insertions, 553 deletions
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java index de6e91e..30e8245 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java @@ -33,10 +33,12 @@ public interface AdminApiFactory { @Singleton static class DefaultAdminApiFactory implements AdminApiFactory { protected final SshHelper sshHelper; + private final GerritRestApi.Factory gerritRestApiFactory; @Inject - public DefaultAdminApiFactory(SshHelper sshHelper) { + public DefaultAdminApiFactory(SshHelper sshHelper, GerritRestApi.Factory gerritRestApiFactory) { this.sshHelper = sshHelper; + this.gerritRestApiFactory = gerritRestApiFactory; } @Override @@ -47,6 +49,8 @@ public interface AdminApiFactory { return Optional.of(new LocalFS(uri)); } else if (isSSH(uri)) { return Optional.of(new RemoteSsh(sshHelper, uri)); + } else if (isGerritHttp(uri)) { + return Optional.of(gerritRestApiFactory.create(uri)); } return Optional.empty(); } @@ -70,4 +74,9 @@ public interface AdminApiFactory { } return false; } + + public static boolean isGerritHttp(URIish uri) { + String scheme = uri.getScheme(); + return scheme != null && scheme.toLowerCase().contains("gerrit+http"); + } } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java index 8b2301b..a43d7d9 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java @@ -14,9 +14,11 @@ package com.googlesource.gerrit.plugins.replication; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Multimap; import com.google.common.flogger.FluentLogger; import com.google.gerrit.common.FileUtil; import com.google.gerrit.extensions.annotations.PluginData; +import com.google.gerrit.extensions.annotations.PluginName; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.git.WorkQueue; @@ -26,12 +28,18 @@ import com.google.inject.Singleton; import java.io.IOException; import java.nio.file.Path; import java.util.List; +import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.transport.URIish; @Singleton public class AutoReloadConfigDecorator implements ReplicationConfig { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final long RELOAD_DELAY = 120; + private static final long RELOAD_INTERVAL = 60; private volatile ReplicationFileBasedConfig currentConfig; private long currentConfigTs; @@ -43,6 +51,8 @@ public class AutoReloadConfigDecorator implements ReplicationConfig { // Use Provider<> instead of injecting the ReplicationQueue because of circular dependency with // ReplicationConfig private final Provider<ReplicationQueue> replicationQueue; + private final ScheduledExecutorService autoReloadExecutor; + private ScheduledFuture<?> autoReloadRunnable; private volatile boolean shuttingDown; @@ -51,7 +61,9 @@ public class AutoReloadConfigDecorator implements ReplicationConfig { SitePaths site, Destination.Factory destinationFactory, Provider<ReplicationQueue> replicationQueue, - @PluginData Path pluginDataDir) + @PluginData Path pluginDataDir, + @PluginName String pluginName, + WorkQueue workQueue) throws ConfigInvalidException, IOException { this.site = site; this.destinationFactory = destinationFactory; @@ -59,6 +71,7 @@ public class AutoReloadConfigDecorator implements ReplicationConfig { this.currentConfig = loadConfig(); this.currentConfigTs = getLastModified(currentConfig); this.replicationQueue = replicationQueue; + this.autoReloadExecutor = workQueue.createQueue(1, pluginName + "_auto-reload-config"); } private static long getLastModified(ReplicationFileBasedConfig cfg) { @@ -81,11 +94,16 @@ public class AutoReloadConfigDecorator implements ReplicationConfig { @Override public synchronized List<Destination> getDestinations(FilterType filterType) { - reloadIfNeeded(); return currentConfig.getDestinations(filterType); } - private void reloadIfNeeded() { + @Override + public synchronized Multimap<Destination, URIish> getURIs( + Optional<String> remoteName, Project.NameKey projectName, FilterType filterType) { + return currentConfig.getURIs(remoteName, projectName, filterType); + } + + private synchronized void reloadIfNeeded() { reload(false); } @@ -136,6 +154,11 @@ public class AutoReloadConfigDecorator implements ReplicationConfig { } @Override + public synchronized int getMaxRefsToLog() { + return currentConfig.getMaxRefsToLog(); + } + + @Override public synchronized boolean isEmpty() { return currentConfig.isEmpty(); } @@ -161,6 +184,10 @@ public class AutoReloadConfigDecorator implements ReplicationConfig { @Override public int shutdown() { this.shuttingDown = true; + if (autoReloadRunnable != null) { + autoReloadRunnable.cancel(false); + autoReloadRunnable = null; + } return currentConfig.shutdown(); } @@ -168,6 +195,9 @@ public class AutoReloadConfigDecorator implements ReplicationConfig { public synchronized void startup(WorkQueue workQueue) { shuttingDown = false; currentConfig.startup(workQueue); + autoReloadRunnable = + autoReloadExecutor.scheduleAtFixedRate( + this::reloadIfNeeded, RELOAD_DELAY, RELOAD_INTERVAL, TimeUnit.SECONDS); } @Override diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java index 29a7ee6..98f364d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.nio.file.Files; import java.util.concurrent.atomic.AtomicReference; import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.transport.CredentialsProvider; public class AutoReloadSecureCredentialsFactoryDecorator implements CredentialsFactory { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); @@ -50,7 +51,7 @@ public class AutoReloadSecureCredentialsFactoryDecorator implements CredentialsF } @Override - public SecureCredentialsProvider create(String remoteName) { + public CredentialsProvider create(String remoteName) { try { if (needsReload()) { secureCredentialsFactory.compareAndSet( diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java new file mode 100644 index 0000000..a8dede3 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java @@ -0,0 +1,70 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; + +import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.reviewdb.client.Project; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType; +import java.util.Optional; +import org.eclipse.jgit.transport.RemoteConfig; +import org.eclipse.jgit.transport.URIish; + +public class CreateProjectTask { + interface Factory { + CreateProjectTask create(Project.NameKey project, String head); + } + + private final RemoteConfig config; + private final ReplicationConfig replicationConfig; + private final DynamicItem<AdminApiFactory> adminApiFactory; + private final Project.NameKey project; + private final String head; + + @Inject + CreateProjectTask( + RemoteConfig config, + ReplicationConfig replicationConfig, + DynamicItem<AdminApiFactory> adminApiFactory, + @Assisted Project.NameKey project, + @Assisted String head) { + this.config = config; + this.replicationConfig = replicationConfig; + this.adminApiFactory = adminApiFactory; + this.project = project; + this.head = head; + } + + public boolean create() { + return replicationConfig + .getURIs(Optional.of(config.getName()), project, FilterType.PROJECT_CREATION).values() + .stream() + .map(u -> createProject(u, project, head)) + .reduce(true, (a, b) -> a && b); + } + + private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) { + Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI); + if (adminApi.isPresent() && adminApi.get().createProject(projectName, head)) { + return true; + } + + repLog.warn("Cannot create new project {} on remote site {}.", projectName, replicateURI); + return false; + } +} diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java index 10719c1..3bb64ab 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java @@ -13,7 +13,9 @@ // limitations under the License. package com.googlesource.gerrit.plugins.replication; +import org.eclipse.jgit.transport.CredentialsProvider; + public interface CredentialsFactory { - SecureCredentialsProvider create(String remoteName); + CredentialsProvider create(String remoteName); } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java new file mode 100644 index 0000000..f9b2ad7 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java @@ -0,0 +1,66 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; + +import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.ioutil.HexFormat; +import com.google.gerrit.server.util.IdGenerator; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import java.util.Optional; +import org.eclipse.jgit.transport.URIish; + +public class DeleteProjectTask implements Runnable { + interface Factory { + DeleteProjectTask create(URIish replicateURI, Project.NameKey project); + } + + private final DynamicItem<AdminApiFactory> adminApiFactory; + private final int id; + private final URIish replicateURI; + private final Project.NameKey project; + + @Inject + DeleteProjectTask( + DynamicItem<AdminApiFactory> adminApiFactory, + IdGenerator ig, + @Assisted URIish replicateURI, + @Assisted Project.NameKey project) { + this.adminApiFactory = adminApiFactory; + this.id = ig.next(); + this.replicateURI = replicateURI; + this.project = project; + } + + @Override + public void run() { + Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI); + if (adminApi.isPresent()) { + adminApi.get().deleteProject(project); + return; + } + + repLog.warn("Cannot delete project {} on remote site {}.", project, replicateURI); + } + + @Override + public String toString() { + return String.format( + "[%s] delete-project %s at %s", HexFormat.fromInt(id), project.get(), replicateURI); + } +} diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java index 522abbd..45e6bf9 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java @@ -15,6 +15,7 @@ package com.googlesource.gerrit.plugins.replication; import static com.googlesource.gerrit.plugins.replication.PushResultProcessing.resolveNodeName; +import static com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig.replaceName; import static org.eclipse.jgit.transport.RemoteRefUpdate.Status.NON_EXISTING; import static org.eclipse.jgit.transport.RemoteRefUpdate.Status.REJECTED_OTHER_REASON; @@ -32,14 +33,12 @@ import com.google.gerrit.reviewdb.client.AccountGroup; import com.google.gerrit.reviewdb.client.Branch; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.client.RefNames; -import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.PluginUser; import com.google.gerrit.server.account.GroupBackend; import com.google.gerrit.server.account.GroupBackends; import com.google.gerrit.server.account.GroupIncludeCache; import com.google.gerrit.server.account.ListGroupMembership; -import com.google.gerrit.server.config.RequestScopedReviewDbProvider; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.git.PerThreadRequestScope; @@ -56,6 +55,7 @@ import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Provider; import com.google.inject.Provides; +import com.google.inject.Scopes; import com.google.inject.assistedinject.Assisted; import com.google.inject.assistedinject.FactoryModuleBuilder; import com.google.inject.servlet.RequestScoped; @@ -74,6 +74,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.function.Function; import org.apache.commons.io.FilenameUtils; +import org.apache.http.impl.client.CloseableHttpClient; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; @@ -97,6 +98,8 @@ public class Destination { private final Map<URIish, PushOne> pending = new HashMap<>(); private final Map<URIish, PushOne> inFlight = new HashMap<>(); private final PushOne.Factory opFactory; + private final DeleteProjectTask.Factory deleteProjectFactory; + private final UpdateHeadTask.Factory updateHeadFactory; private final GitRepositoryManager gitManager; private final PermissionBackend permissionBackend; private final Provider<CurrentUser> userProvider; @@ -173,24 +176,24 @@ public class Destination { bind(Destination.class).toInstance(Destination.this); bind(RemoteConfig.class).toInstance(config.getRemoteConfig()); install(new FactoryModuleBuilder().build(PushOne.Factory.class)); + install(new FactoryModuleBuilder().build(CreateProjectTask.Factory.class)); + install(new FactoryModuleBuilder().build(DeleteProjectTask.Factory.class)); + install(new FactoryModuleBuilder().build(UpdateHeadTask.Factory.class)); + + DynamicItem.itemOf(binder(), AdminApiFactory.class); + DynamicItem.bind(binder(), AdminApiFactory.class) + .to(AdminApiFactory.DefaultAdminApiFactory.class); + + install(new FactoryModuleBuilder().build(GerritRestApi.Factory.class)); + bind(CloseableHttpClient.class) + .toProvider(HttpClientProvider.class) + .in(Scopes.SINGLETON); } @Provides public PerThreadRequestScope.Scoper provideScoper( - final PerThreadRequestScope.Propagator propagator, - final Provider<RequestScopedReviewDbProvider> dbProvider) { - final RequestContext requestContext = - new RequestContext() { - @Override - public CurrentUser getUser() { - return remoteUser; - } - - @Override - public Provider<ReviewDb> getReviewDbProvider() { - return dbProvider.get(); - } - }; + final PerThreadRequestScope.Propagator propagator) { + final RequestContext requestContext = () -> remoteUser; return new PerThreadRequestScope.Scoper() { @Override public <T> Callable<T> scope(Callable<T> callable) { @@ -201,6 +204,8 @@ public class Destination { }); opFactory = child.getInstance(PushOne.Factory.class); + deleteProjectFactory = child.getInstance(DeleteProjectTask.Factory.class); + updateHeadFactory = child.getInstance(UpdateHeadTask.Factory.class); threadScoper = child.getInstance(PerThreadRequestScope.Scoper.class); } @@ -231,24 +236,35 @@ public class Destination { public int shutdown() { int cnt = 0; if (pool != null) { - repLog.warn("Cancelling replication events"); - - foreachPushOp( - pending, - push -> { - push.cancel(); - return null; - }); - pending.clear(); - foreachPushOp( - inFlight, - push -> { - push.setCanceledWhileRunning(); - return null; - }); - inFlight.clear(); - cnt = pool.shutdownNow().size(); - pool = null; + synchronized (stateLock) { + int numPending = pending.size(); + int numInFlight = inFlight.size(); + + if (numPending > 0 || numInFlight > 0) { + repLog.warn( + "Cancelling replication events (pending={}, inFlight={}) for destination {}", + numPending, + numInFlight, + getRemoteConfigName()); + + foreachPushOp( + pending, + push -> { + push.cancel(); + return null; + }); + pending.clear(); + foreachPushOp( + inFlight, + push -> { + push.setCanceledWhileRunning(); + return null; + }); + inFlight.clear(); + } + cnt = pool.shutdownNow().size(); + pool = null; + } } return cnt; } @@ -291,47 +307,44 @@ public class Destination { try { return threadScoper .scope( - new Callable<Boolean>() { - @Override - public Boolean call() throws NoSuchProjectException, PermissionBackendException { - ProjectState projectState; - try { - projectState = projectCache.checkedGet(project); - } catch (IOException e) { - repLog.warn("Error reading project {} from cache", project, e); - return false; - } - if (projectState == null) { - repLog.debug("Project {} does not exist", project); - throw new NoSuchProjectException(project); - } - if (!projectState.statePermitsRead()) { - repLog.debug("Project {} does not permit read", project); - return false; - } - if (!shouldReplicate(projectState, userProvider.get())) { - repLog.debug("Project {} should not be replicated", project); - return false; - } - if (PushOne.ALL_REFS.equals(ref)) { - return true; - } - try { - permissionBackend - .user(userProvider.get()) - .project(project) - .ref(ref) - .check(RefPermission.READ); - } catch (AuthException e) { - repLog.debug( - "Ref {} on project {} is not visible to calling user {}", - ref, - project, - userProvider.get().getUserName().orElse("unknown")); - return false; - } + () -> { + ProjectState projectState; + try { + projectState = projectCache.checkedGet(project); + } catch (IOException e) { + repLog.warn("Error reading project {} from cache", project, e); + return false; + } + if (projectState == null) { + repLog.debug("Project {} does not exist", project); + throw new NoSuchProjectException(project); + } + if (!projectState.statePermitsRead()) { + repLog.debug("Project {} does not permit read", project); + return false; + } + if (!shouldReplicate(projectState, userProvider.get())) { + repLog.debug("Project {} should not be replicated", project); + return false; + } + if (PushOne.ALL_REFS.equals(ref)) { return true; } + try { + permissionBackend + .user(userProvider.get()) + .project(project) + .ref(ref) + .check(RefPermission.READ); + } catch (AuthException e) { + repLog.debug( + "Ref {} on project {} is not visible to calling user {}", + ref, + project, + userProvider.get().getUserName().orElse("unknown")); + return false; + } + return true; }) .call(); } catch (NoSuchProjectException err) { @@ -347,20 +360,17 @@ public class Destination { try { return threadScoper .scope( - new Callable<Boolean>() { - @Override - public Boolean call() throws NoSuchProjectException, PermissionBackendException { - ProjectState projectState; - try { - projectState = projectCache.checkedGet(project); - } catch (IOException e) { - return false; - } - if (projectState == null) { - throw new NoSuchProjectException(project); - } - return shouldReplicate(projectState, userProvider.get()); + () -> { + ProjectState projectState; + try { + projectState = projectCache.checkedGet(project); + } catch (IOException e) { + return false; } + if (projectState == null) { + throw new NoSuchProjectException(project); + } + return shouldReplicate(projectState, userProvider.get()); }) .call(); } catch (NoSuchProjectException err) { @@ -443,6 +453,18 @@ public class Destination { } } + void scheduleDeleteProject(URIish uri, Project.NameKey project) { + @SuppressWarnings("unused") + ScheduledFuture<?> ignored = + pool.schedule(deleteProjectFactory.create(uri, project), 0, TimeUnit.SECONDS); + } + + void scheduleUpdateHead(URIish uri, Project.NameKey project, String newHead) { + @SuppressWarnings("unused") + ScheduledFuture<?> ignored = + pool.schedule(updateHeadFactory.create(uri, project, newHead), 0, TimeUnit.SECONDS); + } + private void addRef(PushOne e, String ref) { e.addRef(ref); postReplicationScheduledEvent(e, ref); @@ -694,8 +716,7 @@ public class Destination { } else if (!remoteNameStyle.equals("slash")) { repLog.debug("Unknown remoteNameStyle: {}, falling back to slash", remoteNameStyle); } - String replacedPath = - ReplicationQueue.replaceName(template.getPath(), name, isSingleProjectMatch()); + String replacedPath = replaceName(template.getPath(), name, isSingleProjectMatch()); return (replacedPath != null) ? template.setPath(replacedPath) : template; } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java new file mode 100644 index 0000000..eac56df --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java @@ -0,0 +1,135 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.googlesource.gerrit.plugins.replication.GerritSshApi.GERRIT_ADMIN_PROTOCOL_PREFIX; +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.gerrit.extensions.restapi.Url; +import com.google.gerrit.reviewdb.client.Project; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import java.io.IOException; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicHeader; +import org.eclipse.jgit.transport.CredentialItem; +import org.eclipse.jgit.transport.RemoteConfig; +import org.eclipse.jgit.transport.URIish; + +public class GerritRestApi implements AdminApi { + + public interface Factory { + GerritRestApi create(URIish uri); + } + + private final CredentialsFactory credentials; + private final CloseableHttpClient httpClient; + private final RemoteConfig remoteConfig; + private final URIish uri; + + @Inject + GerritRestApi( + CredentialsFactory credentials, + CloseableHttpClient httpClient, + RemoteConfig remoteConfig, + @Assisted URIish uri) { + this.credentials = credentials; + this.httpClient = httpClient; + this.remoteConfig = remoteConfig; + this.uri = uri; + } + + @Override + public boolean createProject(Project.NameKey project, String head) { + repLog.info("Creating project {} on {}", project, uri); + String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get())); + try { + return httpClient + .execute(new HttpPut(url), new HttpResponseHandler(), getContext()) + .isSuccessful(); + } catch (IOException e) { + repLog.error("Couldn't perform project creation on {}", uri, e); + return false; + } + } + + @Override + public boolean deleteProject(Project.NameKey project) { + repLog.info("Deleting project {} on {}", project, uri); + String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get())); + try { + httpClient.execute(new HttpDelete(url), new HttpResponseHandler(), getContext()); + return true; + } catch (IOException e) { + repLog.error("Couldn't perform project deletion on {}", uri, e); + } + return false; + } + + @Override + public boolean updateHead(Project.NameKey project, String newHead) { + repLog.info("Updating head of {} on {}", project, uri); + String url = String.format("%s/a/projects/%s/HEAD", toHttpUri(uri), Url.encode(project.get())); + try { + HttpPut req = new HttpPut(url); + req.setEntity(new StringEntity(String.format("{\"ref\": \"%s\"}", newHead), UTF_8.name())); + req.addHeader(new BasicHeader("Content-Type", "application/json")); + httpClient.execute(req, new HttpResponseHandler(), getContext()); + return true; + } catch (IOException e) { + repLog.error("Couldn't perform update head on {}", uri, e); + } + return false; + } + + private HttpClientContext getContext() { + HttpClientContext ctx = HttpClientContext.create(); + ctx.setCredentialsProvider(adapt(credentials.create(remoteConfig.getName()))); + return ctx; + } + + private CredentialsProvider adapt(org.eclipse.jgit.transport.CredentialsProvider cp) { + CredentialItem.Username user = new CredentialItem.Username(); + CredentialItem.Password pass = new CredentialItem.Password(); + if (cp.supports(user, pass) && cp.get(uri, user, pass)) { + CredentialsProvider adapted = new BasicCredentialsProvider(); + adapted.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials(user.getValue(), new String(pass.getValue()))); + return adapted; + } + return null; + } + + private static String toHttpUri(URIish uri) { + String u = uri.toString(); + if (u.startsWith(GERRIT_ADMIN_PROTOCOL_PREFIX)) { + u = u.substring(GERRIT_ADMIN_PROTOCOL_PREFIX.length()); + } + if (u.endsWith("/")) { + return u.substring(0, u.length() - 1); + } + return u; + } +} diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java index ccd8bf4..6dcc80e 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java @@ -28,7 +28,7 @@ public class GerritSshApi implements AdminApi { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); static final int SSH_COMMAND_FAILED = -1; - private static final String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+"; + static final String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+"; protected final SshHelper sshHelper; protected final URIish uri; diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java new file mode 100644 index 0000000..916059c --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java @@ -0,0 +1,108 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.ProvisionException; +import java.nio.file.Files; +import java.nio.file.Path; +import javax.net.ssl.SSLContext; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.HttpClientConnectionManager; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.ssl.SSLContexts; +import org.eclipse.jgit.lib.Config; + +/** Provides an HTTP client with SSL capabilities. */ +class HttpClientProvider implements Provider<CloseableHttpClient> { + private static final int CONNECTIONS_PER_ROUTE = 100; + + // Up to 2 target instances with the max number of connections per host: + private static final int MAX_CONNECTIONS = 2 * CONNECTIONS_PER_ROUTE; + + private static final int MAX_CONNECTION_INACTIVITY = 10000; + private static final int DEFAULT_TIMEOUT_MS = 5000; + + private final Config cfg; + private final SitePaths site; + + @Inject + HttpClientProvider(@GerritServerConfig Config cfg, SitePaths site) { + this.cfg = cfg; + this.site = site; + } + + @Override + public CloseableHttpClient get() { + try { + return HttpClients.custom() + .setConnectionManager(customConnectionManager()) + .setDefaultRequestConfig(customRequestConfig()) + .build(); + } catch (Exception e) { + throw new ProvisionException("Couldn't create CloseableHttpClient", e); + } + } + + private RequestConfig customRequestConfig() { + return RequestConfig.custom() + .setConnectTimeout(DEFAULT_TIMEOUT_MS) + .setSocketTimeout(DEFAULT_TIMEOUT_MS) + .setConnectionRequestTimeout(DEFAULT_TIMEOUT_MS) + .build(); + } + + private HttpClientConnectionManager customConnectionManager() throws Exception { + Registry<ConnectionSocketFactory> socketFactoryRegistry = + RegistryBuilder.<ConnectionSocketFactory>create() + .register("https", buildSslSocketFactory()) + .register("http", PlainConnectionSocketFactory.INSTANCE) + .build(); + PoolingHttpClientConnectionManager connManager = + new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connManager.setDefaultMaxPerRoute(CONNECTIONS_PER_ROUTE); + connManager.setMaxTotal(MAX_CONNECTIONS); + connManager.setValidateAfterInactivity(MAX_CONNECTION_INACTIVITY); + return connManager; + } + + private SSLConnectionSocketFactory buildSslSocketFactory() throws Exception { + String keyStore = cfg.getString("httpd", null, "sslKeyStore"); + if (keyStore == null) { + keyStore = "etc/keystore"; + } + return new SSLConnectionSocketFactory(createSSLContext(site.resolve(keyStore))); + } + + private SSLContext createSSLContext(Path keyStorePath) throws Exception { + SSLContext ctx; + if (Files.exists(keyStorePath)) { + ctx = SSLContexts.custom().loadTrustMaterial(keyStorePath.toFile()).build(); + } else { + ctx = SSLContext.getDefault(); + } + return ctx; + } +} diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java new file mode 100644 index 0000000..595acc7 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java @@ -0,0 +1,68 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.ByteBuffer; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.eclipse.jgit.util.IO; +import org.eclipse.jgit.util.RawParseUtils; + +public class HttpResponse implements AutoCloseable { + + protected CloseableHttpResponse response; + protected Reader reader; + + HttpResponse(CloseableHttpResponse response) { + this.response = response; + } + + public Reader getReader() throws IllegalStateException, IOException { + if (reader == null && response.getEntity() != null) { + reader = new InputStreamReader(response.getEntity().getContent(), UTF_8); + } + return reader; + } + + @Override + public void close() throws IOException { + try { + Reader reader = getReader(); + if (reader != null) { + while (reader.read() != -1) { + // Empty + } + } + } finally { + response.close(); + } + } + + public int getStatusCode() { + return response.getStatusLine().getStatusCode(); + } + + public String getEntityContent() throws IOException { + Preconditions.checkNotNull(response, "Response is not initialized."); + Preconditions.checkNotNull(response.getEntity(), "Response.Entity is not initialized."); + ByteBuffer buf = IO.readWholeStream(response.getEntity().getContent(), 1024); + return RawParseUtils.decode(buf.array(), buf.arrayOffset(), buf.limit()).trim(); + } +} diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java new file mode 100644 index 0000000..6a7c73e --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java @@ -0,0 +1,71 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static javax.servlet.http.HttpServletResponse.SC_CREATED; +import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; +import static javax.servlet.http.HttpServletResponse.SC_OK; + +import com.google.common.flogger.FluentLogger; +import com.googlesource.gerrit.plugins.replication.HttpResponseHandler.HttpResult; +import java.io.IOException; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.ResponseHandler; +import org.apache.http.util.EntityUtils; + +class HttpResponseHandler implements ResponseHandler<HttpResult> { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + static class HttpResult { + private final boolean successful; + private final String message; + + HttpResult(boolean successful, String message) { + this.successful = successful; + this.message = message; + } + + boolean isSuccessful() { + return successful; + } + + String getMessage() { + return message; + } + } + + @Override + public HttpResult handleResponse(HttpResponse response) { + return new HttpResult(isSuccessful(response), parseResponse(response)); + } + + private static boolean isSuccessful(HttpResponse response) { + int sc = response.getStatusLine().getStatusCode(); + return sc == SC_CREATED || sc == SC_NO_CONTENT || sc == SC_OK; + } + + private static String parseResponse(HttpResponse response) { + HttpEntity entity = response.getEntity(); + if (entity != null) { + try { + return EntityUtils.toString(entity); + } catch (IOException e) { + logger.atSevere().withCause(e).log("Error parsing entity"); + } + } + return ""; + } +} diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java index 21a630e..64c8fa1 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java @@ -17,6 +17,7 @@ package com.googlesource.gerrit.plugins.replication; import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toMap; import com.google.common.base.MoreObjects; import com.google.common.base.Throwables; @@ -55,7 +56,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicBoolean; import org.eclipse.jgit.errors.NoRemoteRepositoryException; import org.eclipse.jgit.errors.NotSupportedException; @@ -107,9 +107,9 @@ class PushOne implements ProjectRunnable, CanceledWhileRunning { private final PermissionBackend permissionBackend; private final Destination pool; private final RemoteConfig config; + private final ReplicationConfig replConfig; private final CredentialsProvider credentialsProvider; private final PerThreadRequestScope.Scoper threadScoper; - private final ReplicationQueue replicationQueue; private final Project.NameKey projectName; private final URIish uri; @@ -128,6 +128,7 @@ class PushOne implements ProjectRunnable, CanceledWhileRunning { private final long createdAt; private final ReplicationMetrics metrics; private final ProjectCache projectCache; + private final CreateProjectTask.Factory createProjectFactory; private final AtomicBoolean canceledWhileRunning; private final TransportFactory transportFactory; private DynamicItem<ReplicationPushFilter> replicationPushFilter; @@ -138,13 +139,14 @@ class PushOne implements ProjectRunnable, CanceledWhileRunning { PermissionBackend permissionBackend, Destination p, RemoteConfig c, + ReplicationConfig rc, CredentialsFactory cpFactory, PerThreadRequestScope.Scoper ts, - ReplicationQueue rq, IdGenerator ig, ReplicationStateListeners sl, ReplicationMetrics m, ProjectCache pc, + CreateProjectTask.Factory cpf, TransportFactory tf, @Assisted Project.NameKey d, @Assisted URIish u) { @@ -152,9 +154,9 @@ class PushOne implements ProjectRunnable, CanceledWhileRunning { this.permissionBackend = permissionBackend; pool = p; config = c; + replConfig = rc; credentialsProvider = cpFactory.create(c.getName()); threadScoper = ts; - replicationQueue = rq; projectName = d; uri = u; updateRefRetryCount = 0; @@ -164,6 +166,7 @@ class PushOne implements ProjectRunnable, CanceledWhileRunning { createdAt = System.nanoTime(); metrics = m; projectCache = pc; + createProjectFactory = cpf; canceledWhileRunning = new AtomicBoolean(false); maxRetries = p.getMaxRetries(); transportFactory = tf; @@ -310,12 +313,9 @@ class PushOne implements ProjectRunnable, CanceledWhileRunning { try { threadScoper .scope( - new Callable<Void>() { - @Override - public Void call() { - runPushOperation(); - return null; - } + () -> { + runPushOperation(); + return null; }) .call(); } catch (Exception e) { @@ -439,8 +439,7 @@ class PushOne implements ProjectRunnable, CanceledWhileRunning { if (pool.isCreateMissingRepos()) { try { Ref head = git.exactRef(Constants.HEAD); - if (replicationQueue.createProject( - config.getName(), projectName, head != null ? getName(head) : null)) { + if (createProject(projectName, head != null ? getName(head) : null)) { repLog.warn("Missing repository created; retry replication to {}", uri); pool.reschedule(this, Destination.RetryReason.REPOSITORY_MISSING); } else { @@ -457,6 +456,10 @@ class PushOne implements ProjectRunnable, CanceledWhileRunning { } } + private boolean createProject(Project.NameKey project, String head) { + return createProjectFactory.create(project, head).create(); + } + private String getName(Ref ref) { Ref target = ref; while (target.isSymbolic()) { @@ -486,7 +489,16 @@ class PushOne implements ProjectRunnable, CanceledWhileRunning { return new PushResult(); } - repLog.info("Push to {} references: {}", uri, refUpdatesForLogging(todo)); + if (replConfig.getMaxRefsToLog() == 0 || todo.size() <= replConfig.getMaxRefsToLog()) { + repLog.info("Push to {} references: {}", uri, refUpdatesForLogging(todo)); + } else { + repLog.info( + "Push to {} references (first {} of {} listed): {}", + uri, + replConfig.getMaxRefsToLog(), + todo.size(), + refUpdatesForLogging(todo.subList(0, replConfig.getMaxRefsToLog()))); + } return tn.push(NullProgressMonitor.INSTANCE, todo); } @@ -525,7 +537,8 @@ class PushOne implements ProjectRunnable, CanceledWhileRunning { return Collections.emptyList(); } - Map<String, Ref> local = git.getAllRefs(); + Map<String, Ref> local = + git.getRefDatabase().getRefs().stream().collect(toMap(Ref::getName, r -> r)); boolean filter; PermissionBackend.ForProject forProject = permissionBackend.currentUser().project(projectName); try { diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java index 8542034..92ba4be 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java @@ -15,10 +15,10 @@ package com.googlesource.gerrit.plugins.replication; import com.google.common.flogger.FluentLogger; +import com.google.gerrit.exceptions.StorageException; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.events.RefEvent; import com.google.gerrit.server.permissions.PermissionBackendException; -import com.google.gwtorm.server.OrmException; import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult; import java.lang.ref.WeakReference; import java.util.concurrent.atomic.AtomicBoolean; @@ -202,7 +202,7 @@ public interface PushResultProcessing { private void postEvent(RefEvent event) { try { dispatcher.postEvent(event); - } catch (OrmException | PermissionBackendException e) { + } catch (StorageException | PermissionBackendException e) { logger.atSevere().withCause(e).log("Cannot post event"); } } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java index ee671f5..ccdead8 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java @@ -13,10 +13,12 @@ // limitations under the License. package com.googlesource.gerrit.plugins.replication; +import com.google.common.collect.Multimap; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.git.WorkQueue; import java.nio.file.Path; import java.util.List; +import java.util.Optional; import org.eclipse.jgit.transport.URIish; public interface ReplicationConfig { @@ -31,10 +33,15 @@ public interface ReplicationConfig { List<Destination> getDestinations(FilterType filterType); + Multimap<Destination, URIish> getURIs( + Optional<String> remoteName, Project.NameKey projectName, FilterType filterType); + boolean isReplicateAllOnPluginStart(); boolean isDefaultForceUpdate(); + int getMaxRefsToLog(); + boolean isEmpty(); Path getEventsDirectory(); diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java index 37d63e3..3094929 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java @@ -13,13 +13,21 @@ // limitations under the License. package com.googlesource.gerrit.plugins.replication; +import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerrit; +import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerritHttp; +import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isSSH; +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; import com.google.common.base.Strings; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.SetMultimap; import com.google.common.flogger.FluentLogger; import com.google.gerrit.extensions.annotations.PluginData; import com.google.gerrit.reviewdb.client.Project; @@ -35,6 +43,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import org.eclipse.jgit.errors.ConfigInvalidException; @@ -54,6 +63,7 @@ public class ReplicationFileBasedConfig implements ReplicationConfig { private Path cfgPath; private boolean replicateAllOnPluginStart; private boolean defaultForceUpdate; + private int maxRefsToLog; private int sshCommandTimeout; private int sshConnectionTimeout = DEFAULT_SSH_CONNECTION_TIMEOUT_MS; private final FileBasedConfig config; @@ -130,6 +140,8 @@ public class ReplicationFileBasedConfig implements ReplicationConfig { defaultForceUpdate = config.getBoolean("gerrit", "defaultForceUpdate", false); + maxRefsToLog = config.getInt("gerrit", "maxRefsToLog", 0); + sshCommandTimeout = (int) ConfigUtil.getTimeUnit(config, "gerrit", null, "sshCommandTimeout", 0, SECONDS); sshConnectionTimeout = @@ -180,6 +192,77 @@ public class ReplicationFileBasedConfig implements ReplicationConfig { return dest.build(); } + @Override + public Multimap<Destination, URIish> getURIs( + Optional<String> remoteName, Project.NameKey projectName, FilterType filterType) { + if (getDestinations(filterType).isEmpty()) { + return ImmutableMultimap.of(); + } + + SetMultimap<Destination, URIish> uris = HashMultimap.create(); + for (Destination config : getDestinations(filterType)) { + if (filterType != FilterType.PROJECT_DELETION && !config.wouldPushProject(projectName)) { + continue; + } + + if (remoteName.isPresent() && !config.getRemoteConfigName().equals(remoteName.get())) { + continue; + } + + boolean adminURLUsed = false; + + for (String url : config.getAdminUrls()) { + if (Strings.isNullOrEmpty(url)) { + continue; + } + + URIish uri; + try { + uri = new URIish(url); + } catch (URISyntaxException e) { + repLog.warn("adminURL '{}' is invalid: {}", url, e.getMessage()); + continue; + } + + if (!isGerrit(uri) && !isGerritHttp(uri)) { + String path = + replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch()); + if (path == null) { + repLog.warn("adminURL {} does not contain ${name}", uri); + continue; + } + + uri = uri.setPath(path); + if (!isSSH(uri)) { + repLog.warn("adminURL '{}' is invalid: only SSH and HTTP are supported", uri); + continue; + } + } + uris.put(config, uri); + adminURLUsed = true; + } + + if (!adminURLUsed) { + for (URIish uri : config.getURIs(projectName, "*")) { + uris.put(config, uri); + } + } + } + return uris; + } + + static String replaceName(String in, String name, boolean keyIsOptional) { + String key = "${name}"; + int n = in.indexOf(key); + if (0 <= n) { + return in.substring(0, n) + name + in.substring(n + key.length()); + } + if (keyIsOptional) { + return in; + } + return null; + } + /* (non-Javadoc) * @see com.googlesource.gerrit.plugins.replication.ReplicationConfig#isReplicateAllOnPluginStart() */ @@ -196,6 +279,11 @@ public class ReplicationFileBasedConfig implements ReplicationConfig { return defaultForceUpdate; } + @Override + public int getMaxRefsToLog() { + return maxRefsToLog; + } + private static List<RemoteConfig> allRemotes(FileBasedConfig cfg) throws ConfigInvalidException { Set<String> names = cfg.getSubsections("remote"); List<RemoteConfig> result = Lists.newArrayListWithCapacity(names.size()); diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java index 5fdb375..835d068 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java @@ -22,7 +22,6 @@ import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; import com.google.gerrit.extensions.events.HeadUpdatedListener; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.extensions.events.ProjectDeletedListener; -import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.extensions.registration.DynamicSet; import com.google.gerrit.server.events.EventTypes; import com.google.inject.AbstractModule; @@ -67,10 +66,6 @@ class ReplicationModule extends AbstractModule { EventTypes.register(ReplicationScheduledEvent.TYPE, ReplicationScheduledEvent.class); bind(SshSessionFactory.class).toProvider(ReplicationSshSessionFactoryProvider.class); - DynamicItem.itemOf(binder(), AdminApiFactory.class); - DynamicItem.bind(binder(), AdminApiFactory.class) - .to(AdminApiFactory.DefaultAdminApiFactory.class); - bind(TransportFactory.class).to(TransportFactoryImpl.class).in(Scopes.SINGLETON); } } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java index 22b6cce..51f0c12 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java @@ -14,20 +14,15 @@ package com.googlesource.gerrit.plugins.replication; -import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerrit; -import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isSSH; - -import com.google.common.base.Objects; -import com.google.common.base.Strings; +import com.google.auto.value.AutoValue; import com.google.common.collect.Queues; -import com.google.gerrit.common.Nullable; +import com.google.gerrit.common.UsedAt; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; import com.google.gerrit.extensions.events.HeadUpdatedListener; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.extensions.events.ProjectDeletedListener; import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.reviewdb.client.Project; -import com.google.gerrit.server.UsedAt; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.git.WorkQueue; import com.google.inject.Inject; @@ -35,7 +30,6 @@ import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdat import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType; import com.googlesource.gerrit.plugins.replication.ReplicationTasksStorage.ReplicateRefUpdate; import java.net.URISyntaxException; -import java.util.Collections; import java.util.HashSet; import java.util.Optional; import java.util.Queue; @@ -55,22 +49,9 @@ public class ReplicationQueue private final ReplicationStateListener stateLog; - static String replaceName(String in, String name, boolean keyIsOptional) { - String key = "${name}"; - int n = in.indexOf(key); - if (0 <= n) { - return in.substring(0, n) + name + in.substring(n + key.length()); - } - if (keyIsOptional) { - return in; - } - return null; - } - private final WorkQueue workQueue; private final DynamicItem<EventDispatcher> dispatcher; private final ReplicationConfig config; - private final DynamicItem<AdminApiFactory> adminApiFactory; private final ReplicationTasksStorage replicationTasksStorage; private volatile boolean running; private volatile boolean replaying; @@ -79,7 +60,6 @@ public class ReplicationQueue @Inject ReplicationQueue( WorkQueue wq, - DynamicItem<AdminApiFactory> aaf, ReplicationConfig rc, DynamicItem<EventDispatcher> dis, ReplicationStateListeners sl, @@ -88,7 +68,6 @@ public class ReplicationQueue dispatcher = dis; config = rc; stateLog = sl; - adminApiFactory = aaf; replicationTasksStorage = rts; beforeStartupEventsQueue = Queues.newConcurrentLinkedQueue(); } @@ -152,7 +131,7 @@ public class ReplicationQueue stateLog.warn( "Replication plugin did not finish startup before event, event replication is postponed", state); - beforeStartupEventsQueue.add(new ReferenceUpdatedEvent(projectName, refName)); + beforeStartupEventsQueue.add(ReferenceUpdatedEvent.create(projectName, refName)); return; } @@ -199,6 +178,7 @@ public class ReplicationQueue } private void firePendingEvents() { + replaying = true; try { replaying = true; for (ReplicationTasksStorage.ReplicateRefUpdate t : replicationTasksStorage.list()) { @@ -221,163 +201,39 @@ public class ReplicationQueue @Override public void onProjectDeleted(ProjectDeletedListener.Event event) { - Project.NameKey projectName = new Project.NameKey(event.getProjectName()); - for (URIish uri : getURIs(null, projectName, FilterType.PROJECT_DELETION)) { - deleteProject(uri, projectName); - } + Project.NameKey p = new Project.NameKey(event.getProjectName()); + config.getURIs(Optional.empty(), p, FilterType.PROJECT_DELETION).entries().stream() + .forEach(e -> e.getKey().scheduleDeleteProject(e.getValue(), p)); } @Override public void onHeadUpdated(HeadUpdatedListener.Event event) { - Project.NameKey project = new Project.NameKey(event.getProjectName()); - for (URIish uri : getURIs(null, project, FilterType.ALL)) { - updateHead(uri, project, event.getNewHeadName()); - } + Project.NameKey p = new Project.NameKey(event.getProjectName()); + config.getURIs(Optional.empty(), p, FilterType.ALL).entries().stream() + .forEach(e -> e.getKey().scheduleUpdateHead(e.getValue(), p, event.getNewHeadName())); } private void fireBeforeStartupEvents() { Set<String> eventsReplayed = new HashSet<>(); for (ReferenceUpdatedEvent event : beforeStartupEventsQueue) { - String eventKey = String.format("%s:%s", event.getProjectName(), event.getRefName()); + String eventKey = String.format("%s:%s", event.projectName(), event.refName()); if (!eventsReplayed.contains(eventKey)) { repLog.info("Firing pending task {}", event); - onGitReferenceUpdated(event.getProjectName(), event.getRefName()); + onGitReferenceUpdated(event.projectName(), event.refName()); eventsReplayed.add(eventKey); } } } - private Set<URIish> getURIs( - @Nullable String remoteName, Project.NameKey projectName, FilterType filterType) { - if (config.getDestinations(filterType).isEmpty()) { - return Collections.emptySet(); - } - if (!running) { - repLog.error("Replication plugin did not finish startup before event"); - return Collections.emptySet(); - } - - Set<URIish> uris = new HashSet<>(); - for (Destination config : this.config.getDestinations(filterType)) { - if (filterType != FilterType.PROJECT_DELETION && !config.wouldPushProject(projectName)) { - continue; - } - - if (remoteName != null && !config.getRemoteConfigName().equals(remoteName)) { - continue; - } - - boolean adminURLUsed = false; - - for (String url : config.getAdminUrls()) { - if (Strings.isNullOrEmpty(url)) { - continue; - } - - URIish uri; - try { - uri = new URIish(url); - } catch (URISyntaxException e) { - repLog.warn("adminURL '{}' is invalid: {}", url, e.getMessage()); - continue; - } - - if (!isGerrit(uri)) { - String path = - replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch()); - if (path == null) { - repLog.warn("adminURL {} does not contain ${name}", uri); - continue; - } - - uri = uri.setPath(path); - if (!isSSH(uri)) { - repLog.warn("adminURL '{}' is invalid: only SSH is supported", uri); - continue; - } - } - uris.add(uri); - adminURLUsed = true; - } - - if (!adminURLUsed) { - for (URIish uri : config.getURIs(projectName, "*")) { - uris.add(uri); - } - } - } - return uris; - } - - public boolean createProject(String remoteName, Project.NameKey project, String head) { - boolean success = true; - for (URIish uri : getURIs(remoteName, project, FilterType.PROJECT_CREATION)) { - success &= createProject(uri, project, head); - } - return success; - } + @AutoValue + abstract static class ReferenceUpdatedEvent { - private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) { - Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI); - if (adminApi.isPresent() && adminApi.get().createProject(projectName, head)) { - return true; + static ReferenceUpdatedEvent create(String projectName, String refName) { + return new AutoValue_ReplicationQueue_ReferenceUpdatedEvent(projectName, refName); } - warnCannotPerform("create new project", replicateURI); - return false; - } - - private void deleteProject(URIish replicateURI, Project.NameKey projectName) { - Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI); - if (adminApi.isPresent()) { - adminApi.get().deleteProject(projectName); - return; - } - - warnCannotPerform("delete project", replicateURI); - } - - private void updateHead(URIish replicateURI, Project.NameKey projectName, String newHead) { - Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI); - if (adminApi.isPresent()) { - adminApi.get().updateHead(projectName, newHead); - return; - } + public abstract String projectName(); - warnCannotPerform("update HEAD of project", replicateURI); - } - - private void warnCannotPerform(String op, URIish uri) { - repLog.warn("Cannot {} on remote site {}.", op, uri); - } - - private static class ReferenceUpdatedEvent { - private String projectName; - private String refName; - - public ReferenceUpdatedEvent(String projectName, String refName) { - this.projectName = projectName; - this.refName = refName; - } - - public String getProjectName() { - return projectName; - } - - public String getRefName() { - return refName; - } - - @Override - public int hashCode() { - return Objects.hashCode(projectName, refName); - } - - @Override - public boolean equals(Object obj) { - return (obj instanceof ReferenceUpdatedEvent) - && Objects.equal(projectName, ((ReferenceUpdatedEvent) obj).projectName) - && Objects.equal(refName, ((ReferenceUpdatedEvent) obj).refName); - } + public abstract String refName(); } } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java index e79491c..ed15b92 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java @@ -21,6 +21,8 @@ import java.util.Objects; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.util.FS; /** Looks up a remote's password in secure.config. */ @@ -49,9 +51,9 @@ public class SecureCredentialsFactory implements CredentialsFactory { } @Override - public SecureCredentialsProvider create(String remoteName) { + public CredentialsProvider create(String remoteName) { String user = Objects.toString(config.getString("remote", remoteName, "username"), ""); String pass = Objects.toString(config.getString("remote", remoteName, "password"), ""); - return new SecureCredentialsProvider(user, pass); + return new UsernamePasswordCredentialsProvider(user, pass); } } diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java deleted file mode 100644 index 62b4036..0000000 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (C) 2011 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.googlesource.gerrit.plugins.replication; - -import org.eclipse.jgit.errors.UnsupportedCredentialItem; -import org.eclipse.jgit.transport.CredentialItem; -import org.eclipse.jgit.transport.CredentialsProvider; -import org.eclipse.jgit.transport.URIish; - -/** Looks up a remote's password in secure.config. */ -public class SecureCredentialsProvider extends CredentialsProvider { - private final String cfgUser; - private final String cfgPass; - - SecureCredentialsProvider(String user, String pass) { - cfgUser = user; - cfgPass = pass; - } - - @Override - public boolean isInteractive() { - return false; - } - - @Override - public boolean supports(CredentialItem... items) { - for (CredentialItem i : items) { - if (i instanceof CredentialItem.Username) { - continue; - } else if (i instanceof CredentialItem.Password) { - continue; - } else { - return false; - } - } - return true; - } - - @Override - public boolean get(URIish uri, CredentialItem... items) throws UnsupportedCredentialItem { - String username = uri.getUser(); - if (username == null) { - username = cfgUser; - } - if (username == null) { - return false; - } - - String password = uri.getPass(); - if (password == null) { - password = cfgPass; - } - if (password == null) { - return false; - } - - for (CredentialItem i : items) { - if (i instanceof CredentialItem.Username) { - ((CredentialItem.Username) i).setValue(username); - } else if (i instanceof CredentialItem.Password) { - ((CredentialItem.Password) i).setValue(password.toCharArray()); - } else { - throw new UnsupportedCredentialItem(uri, i.getPromptText()); - } - } - return true; - } -} diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java new file mode 100644 index 0000000..70452b4 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java @@ -0,0 +1,70 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; + +import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.ioutil.HexFormat; +import com.google.gerrit.server.util.IdGenerator; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import java.util.Optional; +import org.eclipse.jgit.transport.URIish; + +public class UpdateHeadTask implements Runnable { + private final DynamicItem<AdminApiFactory> adminApiFactory; + private final int id; + private final URIish replicateURI; + private final Project.NameKey project; + private final String newHead; + + interface Factory { + UpdateHeadTask create(URIish uri, Project.NameKey project, String newHead); + } + + @Inject + UpdateHeadTask( + DynamicItem<AdminApiFactory> adminApiFactory, + IdGenerator ig, + @Assisted URIish replicateURI, + @Assisted Project.NameKey project, + @Assisted String newHead) { + this.adminApiFactory = adminApiFactory; + this.id = ig.next(); + this.replicateURI = replicateURI; + this.project = project; + this.newHead = newHead; + } + + @Override + public void run() { + Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI); + if (adminApi.isPresent()) { + adminApi.get().updateHead(project, newHead); + return; + } + + repLog.warn("Cannot update HEAD of project {} on remote site {}.", project, replicateURI); + } + + @Override + public String toString() { + return String.format( + "[%s] update-head of %s at %s to %s", + HexFormat.fromInt(id), project.get(), replicateURI, newHead); + } +} diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index 0e47010..30d03d2 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md @@ -82,6 +82,10 @@ gerrit.defaultForceUpdate : If true, the default push refspec will be set to use forced update to the remote when no refspec is given. By default, false. +gerrit.maxRefsToLog +: Number of refs, that are pushed during replication, to be logged. + For printing all refs to the logs, use a value of 0. By default, 0. + gerrit.sshCommandTimeout : Timeout for SSH command execution. If 0, there is no timeout and the client waits indefinitely. By default, 0. @@ -191,6 +195,21 @@ remote.NAME.adminUrl local environment. In that case, an alternative SSH url could be specified to repository creation. + To enable replication to different Gerrit instance use + `gerrit+http://` or `gerrit+https://` as protocol name followed + by hostname of another Gerrit server eg. + + `gerrit+http://replica2.my.org/` + <br> + `gerrit+https://replica3.my.org/` + + In this case replication will use Gerrit's REST API + to create/remove projects and update repository HEAD references. + + NOTE: In order to replicate project deletion, the + link:https://gerrit-review.googlesource.com/admin/projects/plugins/delete-project delete-project[delete-project] + plugin must be installed on the other Gerrit. + *Backward compatibility notice* Before Gerrit v2.13 it was possible to enable replication to different @@ -200,10 +219,7 @@ remote.NAME.adminUrl `gerrit+ssh://replica1.my.org/` In that case replication would have used Gerrit's SSH API to - create/remove projects and update repository HEAD references - and, in order to replicate project deletion, the - link:https://gerrit-review.googlesource.com/admin/projects/plugins/delete-project delete-project[delete-project] - plugin was needed to be installed on the other Gerrit. + create/remove projects and update repository HEAD references. The `gerrit+ssh` option is kept for backward compatibility, however the use-case behind it is not valid anymore since the introduction of diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java new file mode 100644 index 0000000..77dc1cc --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java @@ -0,0 +1,116 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.file.Files.createTempDirectory; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.getCurrentArguments; +import static org.easymock.EasyMock.isA; +import static org.easymock.EasyMock.replay; + +import com.google.gerrit.server.config.SitePaths; +import com.google.inject.Injector; +import com.google.inject.Module; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import org.easymock.IAnswer; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.FS; +import org.junit.Before; +import org.junit.Ignore; + +@Ignore +public abstract class AbstractConfigTest { + protected final Path sitePath; + protected final SitePaths sitePaths; + protected final Destination.Factory destinationFactoryMock; + protected final Path pluginDataPath; + + static class FakeDestination extends Destination { + public final DestinationConfiguration config; + + protected FakeDestination(DestinationConfiguration config) { + super(injectorMock(), null, null, null, null, null, null, null, null, null, null, config); + this.config = config; + } + + private static Injector injectorMock() { + Injector injector = createNiceMock(Injector.class); + Injector childInjectorMock = createNiceMock(Injector.class); + expect(injector.createChildInjector((Module) anyObject())).andReturn(childInjectorMock); + replay(childInjectorMock); + replay(injector); + return injector; + } + } + + AbstractConfigTest() throws IOException { + sitePath = createTempPath("site"); + sitePaths = new SitePaths(sitePath); + pluginDataPath = createTempPath("data"); + destinationFactoryMock = createMock(Destination.Factory.class); + } + + @Before + public void setup() { + expect(destinationFactoryMock.create(isA(DestinationConfiguration.class))) + .andAnswer( + new IAnswer<Destination>() { + @Override + public Destination answer() throws Throwable { + return new FakeDestination((DestinationConfiguration) getCurrentArguments()[0]); + } + }) + .anyTimes(); + replay(destinationFactoryMock); + } + + protected static Path createTempPath(String prefix) throws IOException { + return createTempDirectory(prefix); + } + + protected FileBasedConfig newReplicationConfig() { + FileBasedConfig replicationConfig = + new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED); + return replicationConfig; + } + + protected void assertThatIsDestination( + Destination destination, String remoteName, String... remoteUrls) { + DestinationConfiguration destinationConfig = ((FakeDestination) destination).config; + assertThat(destinationConfig.getRemoteConfig().getName()).isEqualTo(remoteName); + assertThat(destinationConfig.getUrls()).containsExactlyElementsIn(remoteUrls); + } + + protected void assertThatContainsDestination( + List<Destination> destinations, String remoteName, String... remoteUrls) { + List<Destination> matchingDestinations = + destinations.stream() + .filter( + (Destination dst) -> + ((FakeDestination) dst).config.getRemoteConfig().getName().equals(remoteName)) + .collect(Collectors.toList()); + + assertThat(matchingDestinations).isNotEmpty(); + + assertThatIsDestination(matchingDestinations.get(0), remoteName, remoteUrls); + } +} diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java new file mode 100644 index 0000000..211cafa --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java @@ -0,0 +1,252 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.google.common.truth.Truth.assertThat; +import static org.easymock.EasyMock.anyInt; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; + +import com.google.gerrit.server.git.WorkQueue; +import com.google.inject.util.Providers; +import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.junit.Before; +import org.junit.Test; + +public class AutoReloadConfigDecoratorTest extends AbstractConfigTest { + private AutoReloadConfigDecorator autoReloadConfig; + private ReplicationQueue replicationQueueMock; + private WorkQueue workQueueMock; + private FakeExecutorService executorService = new FakeExecutorService(); + + public class FakeExecutorService implements ScheduledExecutorService { + public Runnable refreshCommand; + + @Override + public void shutdown() {} + + @Override + public List<Runnable> shutdownNow() { + return null; + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return false; + } + + @Override + public <T> Future<T> submit(Callable<T> task) { + return null; + } + + @Override + public <T> Future<T> submit(Runnable task, T result) { + return null; + } + + @Override + public Future<?> submit(Runnable task) { + return null; + } + + @Override + public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks) + throws InterruptedException { + return null; + } + + @Override + public <T> List<Future<T>> invokeAll( + Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit) + throws InterruptedException { + return null; + } + + @Override + public <T> T invokeAny(Collection<? extends Callable<T>> tasks) + throws InterruptedException, ExecutionException { + return null; + } + + @Override + public <T> T invokeAny(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + return null; + } + + @Override + public void execute(Runnable command) {} + + @Override + public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) { + return null; + } + + @Override + public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit) { + return null; + } + + @Override + public ScheduledFuture<?> scheduleAtFixedRate( + Runnable command, long initialDelay, long period, TimeUnit unit) { + refreshCommand = command; + return null; + } + + @Override + public ScheduledFuture<?> scheduleWithFixedDelay( + Runnable command, long initialDelay, long delay, TimeUnit unit) { + return null; + } + } + + public AutoReloadConfigDecoratorTest() throws IOException { + super(); + } + + @Override + @Before + public void setup() { + super.setup(); + + setupMocks(); + } + + private void setupMocks() { + replicationQueueMock = createNiceMock(ReplicationQueue.class); + expect(replicationQueueMock.isRunning()).andReturn(true); + replay(replicationQueueMock); + + workQueueMock = createNiceMock(WorkQueue.class); + expect(workQueueMock.createQueue(anyInt(), anyObject(String.class))).andReturn(executorService); + replay(workQueueMock); + } + + @Test + public void shouldLoadNotEmptyInitialReplicationConfig() throws Exception { + FileBasedConfig replicationConfig = newReplicationConfig(); + String remoteName = "foo"; + String remoteUrl = "ssh://git@git.somewhere.com/${name}"; + replicationConfig.setString("remote", remoteName, "url", remoteUrl); + replicationConfig.save(); + + autoReloadConfig = + new AutoReloadConfigDecorator( + sitePaths, + destinationFactoryMock, + Providers.of(replicationQueueMock), + pluginDataPath, + "replication", + workQueueMock); + + List<Destination> destinations = autoReloadConfig.getDestinations(FilterType.ALL); + assertThat(destinations).hasSize(1); + assertThatIsDestination(destinations.get(0), remoteName, remoteUrl); + } + + @Test + public void shouldAutoReloadReplicationConfig() throws Exception { + FileBasedConfig replicationConfig = newReplicationConfig(); + replicationConfig.setBoolean("gerrit", null, "autoReload", true); + String remoteName1 = "foo"; + String remoteUrl1 = "ssh://git@git.foo.com/${name}"; + replicationConfig.setString("remote", remoteName1, "url", remoteUrl1); + replicationConfig.save(); + + autoReloadConfig = + new AutoReloadConfigDecorator( + sitePaths, + destinationFactoryMock, + Providers.of(replicationQueueMock), + pluginDataPath, + "replication", + workQueueMock); + autoReloadConfig.startup(workQueueMock); + + List<Destination> destinations = autoReloadConfig.getDestinations(FilterType.ALL); + assertThat(destinations).hasSize(1); + assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1); + + TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS + + String remoteName2 = "bar"; + String remoteUrl2 = "ssh://git@git.bar.com/${name}"; + replicationConfig.setString("remote", remoteName2, "url", remoteUrl2); + replicationConfig.save(); + executorService.refreshCommand.run(); + + destinations = autoReloadConfig.getDestinations(FilterType.ALL); + assertThat(destinations).hasSize(2); + assertThatContainsDestination(destinations, remoteName1, remoteUrl1); + assertThatContainsDestination(destinations, remoteName2, remoteUrl2); + } + + @Test + public void shouldNotAutoReloadReplicationConfigIfDisabled() throws Exception { + String remoteName1 = "foo"; + String remoteUrl1 = "ssh://git@git.foo.com/${name}"; + FileBasedConfig replicationConfig = newReplicationConfig(); + replicationConfig.setBoolean("gerrit", null, "autoReload", false); + replicationConfig.setString("remote", remoteName1, "url", remoteUrl1); + replicationConfig.save(); + + autoReloadConfig = + new AutoReloadConfigDecorator( + sitePaths, + destinationFactoryMock, + Providers.of(replicationQueueMock), + pluginDataPath, + "replication", + workQueueMock); + autoReloadConfig.startup(workQueueMock); + + List<Destination> destinations = autoReloadConfig.getDestinations(FilterType.ALL); + assertThat(destinations).hasSize(1); + assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1); + + TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS + + replicationConfig.setString("remote", "bar", "url", "ssh://git@git.bar.com/${name}"); + replicationConfig.save(); + executorService.refreshCommand.run(); + + assertThat(autoReloadConfig.getDestinations(FilterType.ALL)).isEqualTo(destinations); + } +} diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java index 5fa7b98..0f6d629 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java @@ -15,20 +15,13 @@ package com.googlesource.gerrit.plugins.replication; import static org.easymock.EasyMock.createMock; -import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.verify; -import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.permissions.PermissionBackendException; -import com.google.gwtorm.client.KeyUtil; -import com.google.gwtorm.server.OrmException; -import com.google.gwtorm.server.SchemaFactory; -import com.google.gwtorm.server.StandardKeyEncoder; import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing; import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult; import java.net.URISyntaxException; @@ -37,12 +30,7 @@ import org.eclipse.jgit.transport.URIish; import org.junit.Before; import org.junit.Test; -@SuppressWarnings("unchecked") public class GitUpdateProcessingTest { - static { - KeyUtil.setEncoderImpl(new StandardKeyEncoder()); - } - private EventDispatcher dispatcherMock; private GitUpdateProcessing gitUpdateProcessing; @@ -50,17 +38,11 @@ public class GitUpdateProcessingTest { public void setUp() throws Exception { dispatcherMock = createMock(EventDispatcher.class); replay(dispatcherMock); - ReviewDb reviewDbMock = createNiceMock(ReviewDb.class); - replay(reviewDbMock); - SchemaFactory<ReviewDb> schemaMock = createMock(SchemaFactory.class); - expect(schemaMock.open()).andReturn(reviewDbMock).anyTimes(); - replay(schemaMock); gitUpdateProcessing = new GitUpdateProcessing(dispatcherMock); } @Test - public void headRefReplicated() - throws URISyntaxException, OrmException, PermissionBackendException { + public void headRefReplicated() throws URISyntaxException, PermissionBackendException { reset(dispatcherMock); RefReplicatedEvent expectedEvent = new RefReplicatedEvent( @@ -83,8 +65,7 @@ public class GitUpdateProcessingTest { } @Test - public void changeRefReplicated() - throws URISyntaxException, OrmException, PermissionBackendException { + public void changeRefReplicated() throws URISyntaxException, PermissionBackendException { reset(dispatcherMock); RefReplicatedEvent expectedEvent = new RefReplicatedEvent( @@ -107,7 +88,7 @@ public class GitUpdateProcessingTest { } @Test - public void onAllNodesReplicated() throws OrmException, PermissionBackendException { + public void onAllNodesReplicated() throws PermissionBackendException { reset(dispatcherMock); RefReplicationDoneEvent expectedDoneEvent = new RefReplicationDoneEvent("someProject", "refs/heads/master", 5); diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java index af065b3..c010ddb 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java @@ -53,6 +53,7 @@ import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.storage.file.FileBasedConfig; @@ -82,7 +83,6 @@ public class PushOneTest { private RefSpec refSpecMock; private CredentialsFactory credentialsFactory; private PerThreadRequestScope.Scoper threadRequestScoperMock; - private ReplicationQueue replicationQueueMock; private IdGenerator idGeneratorMock; private ReplicationStateListeners replicationStateListenersMock; private ReplicationMetrics replicationMetricsMock; @@ -94,10 +94,13 @@ public class PushOneTest { private PushConnection pushConnection; private ProjectState projectStateMock; private RefUpdate refUpdateMock; + private CreateProjectTask.Factory createProjectTaskFactoryMock; + private ReplicationConfig replicationConfigMock; + private RefDatabase refDatabaseMock; private Project.NameKey projectNameKey; private URIish urish; - private Map<String, Ref> localRefs; + private List<Ref> localRefs; private Map<String, Ref> remoteRefs; private CountDownLatch isCallFinished; @@ -112,8 +115,7 @@ public class PushOneTest { new ObjectIdRef.Unpeeled( NEW, "foo", ObjectId.fromString("0000000000000000000000000000000000000001")); - localRefs = new HashMap<>(); - localRefs.put("fooProject", newLocalRef); + localRefs = Arrays.asList(newLocalRef); Ref remoteRef = new ObjectIdRef.Unpeeled(NEW, "foo", ObjectId.zeroId()); remoteRefs = new HashMap<>(); @@ -147,7 +149,6 @@ public class PushOneTest { setupFetchConnectionMock(); setupPushConnectionMock(); setupRequestScopeMock(); - replicationQueueMock = createNiceMock(ReplicationQueue.class); idGeneratorMock = createNiceMock(IdGenerator.class); replicationStateListenersMock = createNiceMock(ReplicationStateListeners.class); @@ -158,6 +159,8 @@ public class PushOneTest { setupProjectCacheMock(); + replicationConfigMock = createNiceMock(ReplicationConfig.class); + replay( gitRepositoryManagerMock, refUpdateMock, @@ -167,7 +170,6 @@ public class PushOneTest { remoteConfigMock, credentialsFactory, threadRequestScoperMock, - replicationQueueMock, idGeneratorMock, replicationStateListenersMock, replicationMetricsMock, @@ -179,7 +181,9 @@ public class PushOneTest { forProjectMock, fetchConnection, pushConnection, - refSpecMock); + refSpecMock, + refDatabaseMock, + replicationConfigMock); } @Test @@ -260,13 +264,14 @@ public class PushOneTest { permissionBackendMock, destinationMock, remoteConfigMock, + replicationConfigMock, credentialsFactory, threadRequestScoperMock, - replicationQueueMock, idGeneratorMock, replicationStateListenersMock, replicationMetricsMock, projectCacheMock, + createProjectTaskFactoryMock, transportFactoryMock, projectNameKey, urish); @@ -358,11 +363,12 @@ public class PushOneTest { expect(gitRepositoryManagerMock.openRepository(projectNameKey)).andReturn(repositoryMock); } - @SuppressWarnings("deprecation") private void setupRepositoryMock(FileBasedConfig config) throws IOException { repositoryMock = createNiceMock(Repository.class); + refDatabaseMock = createNiceMock(RefDatabase.class); expect(repositoryMock.getConfig()).andReturn(config).anyTimes(); - expect(repositoryMock.getAllRefs()).andReturn(localRefs); + expect(repositoryMock.getRefDatabase()).andReturn(refDatabaseMock); + expect(refDatabaseMock.getRefs()).andReturn(localRefs); expect(repositoryMock.updateRef("fooProject")).andReturn(refUpdateMock); } diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationDaemon.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationDaemon.java index 29908b5..d9389b2 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationDaemon.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationDaemon.java @@ -20,6 +20,7 @@ import com.google.common.flogger.FluentLogger; import com.google.gerrit.acceptance.LightweightPluginDaemonTest; import com.google.gerrit.acceptance.TestPlugin; import com.google.gerrit.acceptance.UseLocalDisk; +import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.config.SitePaths; import com.google.inject.Inject; @@ -60,6 +61,7 @@ public class ReplicationDaemon extends LightweightPluginDaemonTest { + TEST_PROJECT_CREATION_SECONDS); @Inject protected SitePaths sitePaths; + @Inject private ProjectOperations projectOperations; protected Path gitPath; protected FileBasedConfig config; @@ -79,7 +81,14 @@ public class ReplicationDaemon extends LightweightPluginDaemonTest { protected void setReplicationDestination( String remoteName, String replicaSuffix, Optional<String> project) throws IOException { setReplicationDestination( - remoteName, Arrays.asList(replicaSuffix), project, TEST_REPLICATION_RETRY_MINUTES); + remoteName, Arrays.asList(replicaSuffix), project, TEST_REPLICATION_DELAY_SECONDS); + } + + protected void setReplicationDestination( + String remoteName, String replicaSuffix, Optional<String> project, boolean mirror) + throws IOException { + setReplicationDestination( + remoteName, Arrays.asList(replicaSuffix), project, TEST_REPLICATION_DELAY_SECONDS, mirror); } protected void setReplicationDestination( @@ -88,12 +97,33 @@ public class ReplicationDaemon extends LightweightPluginDaemonTest { setReplicationDestination(remoteName, Arrays.asList(replicaSuffix), project, replicationDelay); } - protected FileBasedConfig setReplicationDestination( + protected void setReplicationDestination( String remoteName, List<String> replicaSuffixes, Optional<String> project, int replicationDelay) throws IOException { + setReplicationDestination(remoteName, replicaSuffixes, project, replicationDelay, false); + } + + protected void setReplicationDestination( + String remoteName, + String replicaSuffix, + Optional<String> project, + int replicationDelay, + boolean mirror) + throws IOException { + setReplicationDestination( + remoteName, Arrays.asList(replicaSuffix), project, replicationDelay, mirror); + } + + protected FileBasedConfig setReplicationDestination( + String remoteName, + List<String> replicaSuffixes, + Optional<String> project, + int replicationDelay, + boolean mirror) + throws IOException { List<String> replicaUrls = replicaSuffixes.stream() .map(suffix -> gitPath.resolve("${name}" + suffix + ".git").toString()) @@ -101,13 +131,14 @@ public class ReplicationDaemon extends LightweightPluginDaemonTest { config.setStringList("remote", remoteName, "url", replicaUrls); config.setInt("remote", remoteName, "replicationDelay", replicationDelay); config.setInt("remote", remoteName, "replicationRetry", TEST_REPLICATION_RETRY_MINUTES); + config.setBoolean("remote", remoteName, "mirror", mirror); project.ifPresent(prj -> config.setString("remote", remoteName, "projects", prj)); config.save(); return config; } protected Project.NameKey createTestProject(String name) throws Exception { - return createProject(name); + return projectOperations.newProject().name(name).create(); } protected boolean isPushCompleted(Project.NameKey project, String ref, Duration timeOut) { diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java new file mode 100644 index 0000000..36cc209 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java @@ -0,0 +1,72 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.google.common.truth.Truth.assertThat; + +import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType; +import java.io.IOException; +import java.util.List; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.junit.Test; + +public class ReplicationFileBasedConfigTest extends AbstractConfigTest { + + public ReplicationFileBasedConfigTest() throws IOException { + super(); + } + + @Test + public void shouldLoadOneDestination() throws Exception { + String remoteName = "foo"; + String remoteUrl = "ssh://git@git.somewhere.com/${name}"; + FileBasedConfig config = newReplicationConfig(); + config.setString("remote", remoteName, "url", remoteUrl); + config.save(); + + ReplicationFileBasedConfig replicationConfig = newReplicationFileBasedConfig(); + List<Destination> destinations = replicationConfig.getDestinations(FilterType.ALL); + assertThat(destinations).hasSize(1); + + assertThatIsDestination(destinations.get(0), remoteName, remoteUrl); + } + + @Test + public void shouldLoadTwoDestinations() throws Exception { + String remoteName1 = "foo"; + String remoteUrl1 = "ssh://git@git.somewhere.com/${name}"; + String remoteName2 = "bar"; + String remoteUrl2 = "ssh://git@git.elsewhere.com/${name}"; + FileBasedConfig config = newReplicationConfig(); + config.setString("remote", remoteName1, "url", remoteUrl1); + config.setString("remote", remoteName2, "url", remoteUrl2); + config.save(); + + ReplicationFileBasedConfig replicationConfig = newReplicationFileBasedConfig(); + List<Destination> destinations = replicationConfig.getDestinations(FilterType.ALL); + assertThat(destinations).hasSize(2); + + assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1); + assertThatIsDestination(destinations.get(1), remoteName2, remoteUrl2); + } + + private ReplicationFileBasedConfig newReplicationFileBasedConfig() + throws ConfigInvalidException, IOException { + ReplicationFileBasedConfig replicationConfig = + new ReplicationFileBasedConfig(sitePaths, destinationFactoryMock, pluginDataPath); + return replicationConfig; + } +} diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java index a6ccbec..b1d068e 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java @@ -78,7 +78,7 @@ public class ReplicationIT extends ReplicationDaemon { new ProjectDeletedListener.Event() { @Override public String getProjectName() { - return name(projectNameDeleted); + return projectNameDeleted; } @Override @@ -96,7 +96,7 @@ public class ReplicationIT extends ReplicationDaemon { @Test public void shouldReplicateNewChangeRef() throws Exception { - Project.NameKey targetProject = createTestProject("projectreplica"); + Project.NameKey targetProject = createTestProject(project + "replica"); setReplicationDestination("foo", "replica", ALL_PROJECTS); reloadConfig(); @@ -119,7 +119,7 @@ public class ReplicationIT extends ReplicationDaemon { setReplicationDestination("foo", "replica", ALL_PROJECTS); reloadConfig(); - Project.NameKey targetProject = createTestProject("projectreplica"); + Project.NameKey targetProject = createTestProject(project + "replica"); String newBranch = "refs/heads/mybranch"; String master = "refs/heads/master"; BranchInput input = new BranchInput(); @@ -139,8 +139,8 @@ public class ReplicationIT extends ReplicationDaemon { @Test public void shouldReplicateNewBranchToTwoRemotes() throws Exception { - Project.NameKey targetProject1 = createTestProject("projectreplica1"); - Project.NameKey targetProject2 = createTestProject("projectreplica2"); + Project.NameKey targetProject1 = createTestProject(project + "replica1"); + Project.NameKey targetProject2 = createTestProject(project + "replica2"); setReplicationDestination("foo1", "replica1", ALL_PROJECTS); setReplicationDestination("foo2", "replica2", ALL_PROJECTS); @@ -168,7 +168,7 @@ public class ReplicationIT extends ReplicationDaemon { @Test public void shouldMatchTemplatedURL() throws Exception { - Project.NameKey targetProject = createTestProject("projectreplica"); + Project.NameKey targetProject = createTestProject(project + "replica"); setReplicationDestination("foo", "replica", ALL_PROJECTS); reloadConfig(); @@ -194,7 +194,7 @@ public class ReplicationIT extends ReplicationDaemon { @Test public void shouldMatchRealURL() throws Exception { - Project.NameKey targetProject = createTestProject("projectreplica"); + Project.NameKey targetProject = createTestProject(project + "replica"); setReplicationDestination("foo", "replica", ALL_PROJECTS); reloadConfig(); @@ -223,7 +223,7 @@ public class ReplicationIT extends ReplicationDaemon { setReplicationDestination("foo", "replica", ALL_PROJECTS); reloadConfig(); - Project.NameKey targetProject = createTestProject("projectreplica"); + Project.NameKey targetProject = createTestProject(project + "replica"); String newHead = "refs/heads/newhead"; String master = "refs/heads/master"; BranchInput input = new BranchInput(); @@ -241,9 +241,50 @@ public class ReplicationIT extends ReplicationDaemon { } @Test + public void shouldReplicateBranchDeletionWhenMirror() throws Exception { + replicateBranchDeletion(true); + } + + @Test + public void shouldNotReplicateBranchDeletionWhenNotMirror() throws Exception { + replicateBranchDeletion(false); + } + + private void replicateBranchDeletion(boolean mirror) throws Exception { + setReplicationDestination("foo", "replica", ALL_PROJECTS, mirror); + reloadConfig(); + + Project.NameKey targetProject = createTestProject(project + "replica"); + String branchToDelete = "refs/heads/todelete"; + String master = "refs/heads/master"; + BranchInput input = new BranchInput(); + input.revision = master; + gApi.projects().name(project.get()).branch(branchToDelete).create(input); + + try (Repository repo = repoManager.openRepository(targetProject)) { + waitUntil(() -> checkedGetRef(repo, branchToDelete) != null); + } + + gApi.projects().name(project.get()).branch(branchToDelete).delete(); + + try (Repository repo = repoManager.openRepository(targetProject)) { + if (mirror) { + waitUntil(() -> checkedGetRef(repo, branchToDelete) == null); + } + + Ref targetBranchRef = getRef(repo, branchToDelete); + if (mirror) { + assertThat(targetBranchRef).isNull(); + } else { + assertThat(targetBranchRef).isNotNull(); + } + } + } + + @Test public void shouldNotDrainTheQueueWhenReloading() throws Exception { // Setup repo to replicate - Project.NameKey targetProject = createTestProject("projectreplica"); + Project.NameKey targetProject = createTestProject(project + "replica"); String remoteName = "doNotDrainQueue"; setReplicationDestination(remoteName, "replica", ALL_PROJECTS); @@ -265,7 +306,7 @@ public class ReplicationIT extends ReplicationDaemon { @Test public void shouldDrainTheQueueWhenReloading() throws Exception { // Setup repo to replicate - Project.NameKey targetProject = createTestProject("projectreplica"); + Project.NameKey targetProject = createTestProject(project + "replica"); String remoteName = "drainQueue"; setReplicationDestination(remoteName, "replica", ALL_PROJECTS); @@ -287,6 +328,28 @@ public class ReplicationIT extends ReplicationDaemon { } } + @Test + public void shouldNotDropEventsWhenStarting() throws Exception { + Project.NameKey targetProject = createTestProject(project + "replica"); + + setReplicationDestination("foo", "replica", ALL_PROJECTS); + reloadConfig(); + + replicationQueueStop(); + Result pushResult = createChange(); + replicationQueueStart(); + + RevCommit sourceCommit = pushResult.getCommit(); + String sourceRef = pushResult.getPatchSet().getRefName(); + + try (Repository repo = repoManager.openRepository(targetProject)) { + waitUntil(() -> checkedGetRef(repo, sourceRef) != null); + Ref targetBranchRef = getRef(repo, sourceRef); + assertThat(targetBranchRef).isNotNull(); + assertThat(targetBranchRef.getObjectId()).isEqualTo(sourceCommit.getId()); + } + } + private Ref getRef(Repository repo, String branchName) throws IOException { return repo.getRefDatabase().exactRef(branchName); } @@ -302,7 +365,27 @@ public class ReplicationIT extends ReplicationDaemon { } private void shutdownConfig() { - plugin.getSysInjector().getInstance(AutoReloadConfigDecorator.class).shutdown(); + getAutoReloadConfigDecoratorInstance().shutdown(); + } + + private void replicationQueueStart() { + getReplicationQueueInstance().start(); + } + + private void replicationQueueStop() { + getReplicationQueueInstance().stop(); + } + + private AutoReloadConfigDecorator getAutoReloadConfigDecoratorInstance() { + return getInstance(AutoReloadConfigDecorator.class); + } + + private ReplicationQueue getReplicationQueueInstance() { + return getInstance(ReplicationQueue.class); + } + + private <T> T getInstance(Class<T> classObj) { + return plugin.getSysInjector().getInstance(classObj); } private ObjectId createNewBranchWithoutPush(String fromBranch, String newBranch) diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationQueueIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationQueueIT.java deleted file mode 100644 index 1a5c6a1..0000000 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationQueueIT.java +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.googlesource.gerrit.plugins.replication; - -import static com.google.common.truth.Truth.assertThat; -import static java.util.stream.Collectors.toList; - -import com.google.common.base.Stopwatch; -import com.google.common.flogger.FluentLogger; -import com.google.gerrit.acceptance.LightweightPluginDaemonTest; -import com.google.gerrit.acceptance.PushOneCommit.Result; -import com.google.gerrit.acceptance.TestPlugin; -import com.google.gerrit.acceptance.UseLocalDisk; -import com.google.gerrit.reviewdb.client.Project; -import com.google.gerrit.server.config.SitePaths; -import com.google.inject.Inject; -import java.io.IOException; -import java.nio.file.Path; -import java.time.Duration; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.storage.file.FileBasedConfig; -import org.eclipse.jgit.util.FS; -import org.junit.Test; - -@UseLocalDisk -@TestPlugin( - name = "replication", - sysModule = "com.googlesource.gerrit.plugins.replication.ReplicationModule") -public class ReplicationQueueIT extends LightweightPluginDaemonTest { - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - - private static final int TEST_REPLICATION_DELAY = 1; - private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2); - - @Inject private SitePaths sitePaths; - private Path gitPath; - private FileBasedConfig config; - - @Override - public void setUpTestPlugin() throws Exception { - gitPath = sitePaths.site_path.resolve("git"); - config = - new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED); - - setReplicationDestination("foo", "replica"); - super.setUpTestPlugin(); - } - - @Test - public void shouldNotDropEventsWhenStarting() throws Exception { - Project.NameKey targetProject = createProject("projectreplica"); - - replicationQueueStop(); - Result pushResult = createChange(); - replicationQueueStart(); - - RevCommit sourceCommit = pushResult.getCommit(); - String sourceRef = pushResult.getPatchSet().getRefName(); - - try (Repository repo = repoManager.openRepository(targetProject)) { - waitUntil(() -> checkedGetRef(repo, sourceRef) != null); - Ref targetBranchRef = getRef(repo, sourceRef); - assertThat(targetBranchRef).isNotNull(); - assertThat(targetBranchRef.getObjectId()).isEqualTo(sourceCommit.getId()); - } - } - - private Ref getRef(Repository repo, String branchName) throws IOException { - return repo.getRefDatabase().exactRef(branchName); - } - - private Ref checkedGetRef(Repository repo, String branchName) { - try { - return repo.getRefDatabase().exactRef(branchName); - } catch (Exception e) { - logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo); - return null; - } - } - - private void setReplicationDestination(String remoteName, String replicaSuffix) - throws IOException { - setReplicationDestination(remoteName, Arrays.asList(replicaSuffix)); - } - - private void setReplicationDestination(String remoteName, List<String> replicaSuffixes) - throws IOException { - - List<String> replicaUrls = - replicaSuffixes.stream() - .map(suffix -> gitPath.resolve("${name}" + suffix + ".git").toString()) - .collect(toList()); - config.setStringList("remote", remoteName, "url", replicaUrls); - config.setInt("remote", remoteName, "replicationDelay", TEST_REPLICATION_DELAY); - config.save(); - } - - private void waitUntil(Supplier<Boolean> waitCondition) throws InterruptedException { - Stopwatch stopwatch = Stopwatch.createStarted(); - while (!waitCondition.get()) { - if (stopwatch.elapsed().compareTo(TEST_TIMEOUT) > 0) { - throw new InterruptedException(); - } - TimeUnit.MILLISECONDS.sleep(50); - } - } - - private void replicationQueueStart() { - plugin.getSysInjector().getInstance(ReplicationQueue.class).start(); - } - - private void replicationQueueStop() { - plugin.getSysInjector().getInstance(ReplicationQueue.class).stop(); - } -} diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStorageIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStorageIT.java index 9392f0d..a0a0e31 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStorageIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStorageIT.java @@ -20,6 +20,7 @@ import static java.util.stream.Collectors.toList; import com.google.gerrit.acceptance.TestPlugin; import com.google.gerrit.acceptance.UseLocalDisk; +import com.google.gerrit.extensions.api.projects.BranchInput; import com.google.gerrit.reviewdb.client.Project; import com.googlesource.gerrit.plugins.replication.ReplicationTasksStorage.ReplicateRefUpdate; import java.time.Duration; @@ -46,7 +47,7 @@ public class ReplicationStorageIT extends ReplicationDaemon { protected static final int TEST_REPLICATION_MAX_RETRIES = 1; protected static final Duration TEST_TASK_FINISH_TIMEOUT = Duration.ofSeconds(TEST_TASK_FINISH_SECONDS); - private ReplicationTasksStorage tasksStorage; + protected ReplicationTasksStorage tasksStorage; @Override public void setUpTestPlugin() throws Exception { @@ -57,8 +58,8 @@ public class ReplicationStorageIT extends ReplicationDaemon { @Test public void shouldCreateIndividualReplicationTasksForEveryRemoteUrlPair() throws Exception { List<String> replicaSuffixes = Arrays.asList("replica1", "replica2"); - createTestProject("projectreplica1"); - createTestProject("projectreplica2"); + createTestProject(project + "replica1"); + createTestProject(project + "replica2"); setReplicationDestination("foo1", replicaSuffixes, ALL_PROJECTS, Integer.MAX_VALUE); setReplicationDestination("foo2", replicaSuffixes, ALL_PROJECTS, Integer.MAX_VALUE); @@ -71,7 +72,7 @@ public class ReplicationStorageIT extends ReplicationDaemon { @Test public void shouldCreateOneReplicationTaskWhenSchedulingRepoFullSync() throws Exception { - createTestProject("projectreplica"); + createTestProject(project + "replica"); setReplicationDestination("foo", "replica", ALL_PROJECTS, Integer.MAX_VALUE); reloadConfig(); @@ -106,8 +107,8 @@ public class ReplicationStorageIT extends ReplicationDaemon { public void shouldFireAndCompletePendingOnlyToIncompleteUri() throws Exception { String suffix1 = "replica1"; String suffix2 = "replica2"; - Project.NameKey target1 = createTestProject("project" + suffix1); - Project.NameKey target2 = createTestProject("project" + suffix2); + Project.NameKey target1 = createTestProject(project + suffix1); + Project.NameKey target2 = createTestProject(project + suffix2); String remote1 = "foo1"; String remote2 = "foo2"; setReplicationDestination(remote1, suffix1, ALL_PROJECTS, Integer.MAX_VALUE); @@ -149,8 +150,8 @@ public class ReplicationStorageIT extends ReplicationDaemon { public void shouldFireAndCompletePendingChangeRefs() throws Exception { String suffix1 = "replica1"; String suffix2 = "replica2"; - Project.NameKey target1 = createTestProject("project" + suffix1); - Project.NameKey target2 = createTestProject("project" + suffix2); + Project.NameKey target1 = createTestProject(project + suffix1); + Project.NameKey target2 = createTestProject(project + suffix2); String remote1 = "foo1"; String remote2 = "foo2"; setReplicationDestination(remote1, suffix1, ALL_PROJECTS, Integer.MAX_VALUE); @@ -177,7 +178,7 @@ public class ReplicationStorageIT extends ReplicationDaemon { @Test public void shouldMatchTemplatedUrl() throws Exception { - createTestProject("projectreplica"); + createTestProject(project + "replica"); setReplicationDestination("foo", "replica", ALL_PROJECTS, Integer.MAX_VALUE); reloadConfig(); @@ -199,7 +200,7 @@ public class ReplicationStorageIT extends ReplicationDaemon { @Test public void shouldMatchRealUrl() throws Exception { - createTestProject("projectreplica"); + createTestProject(project + "replica"); setReplicationDestination("foo", "replica", ALL_PROJECTS, Integer.MAX_VALUE); reloadConfig(); @@ -220,6 +221,16 @@ public class ReplicationStorageIT extends ReplicationDaemon { } @Test + public void shouldReplicateBranchDeletionWhenMirror() throws Exception { + replicateBranchDeletion(true); + } + + @Test + public void shouldNotReplicateBranchDeletionWhenNotMirror() throws Exception { + replicateBranchDeletion(false); + } + + @Test public void shouldCleanupTasksAfterNewProjectReplication() throws Exception { setReplicationDestination("task_cleanup_project", "replica", ALL_PROJECTS); config.setInt("remote", "task_cleanup_project", "replicationRetry", 0); @@ -234,6 +245,26 @@ public class ReplicationStorageIT extends ReplicationDaemon { WaitUtil.waitUntil(() -> tasksStorage.list().size() == 0, TEST_TASK_FINISH_TIMEOUT); } + private void replicateBranchDeletion(boolean mirror) throws Exception { + setReplicationDestination("foo", "replica", ALL_PROJECTS); + reloadConfig(); + + Project.NameKey targetProject = createTestProject(project + "replica"); + String branchToDelete = "refs/heads/todelete"; + String master = "refs/heads/master"; + BranchInput input = new BranchInput(); + input.revision = master; + gApi.projects().name(project.get()).branch(branchToDelete).create(input); + isPushCompleted(targetProject, branchToDelete, TEST_PUSH_TIMEOUT); + + setReplicationDestination("foo", "replica", ALL_PROJECTS, Integer.MAX_VALUE, mirror); + reloadConfig(); + + gApi.projects().name(project.get()).branch(branchToDelete).delete(); + + assertThat(listReplicationTasks(branchToDelete)).hasSize(1); + } + private Stream<ReplicateRefUpdate> changeReplicationTasksForRemote( String changeRef, String remote) { return tasksStorage.list().stream() |