Add support for direct replication to another Gerrit instance

Currently when replicating to another Gerrit instance replication plugin
requires also system SSH access in order to create, delete and update
HEAD reference of replicated project. For those three operations plugin
will execute system commands over SSH.

When other site of replication is also a Gerrit server we can use its
SSH API to perform operations mentioned above. Then we drop dependency
on system SSH access.

In order to set up a SSH connection between two Gerrit instances, one
must generate ssh-key-pair with empty passphrase. Then this key-pair can
be set to be used for connecting to explicitly configured destination
host.

TEST PLAN:
To test this patch it is recommended to generate new ssh-key-pair and
configure the ssh client like follows:

 * ssh-keygen -f ~/.ssh/empty-passphrase
 * echo "Host localhost\n\tPort 29419\n\tIdentityFile ~/.ssh/empty-passphrase" >> ~/.ssh/config

Then set up a second Gerrit instance (set up sshd to listen on port
29419) which will be the replication destination and create
replication user with public ssh-key from ~/.ssh/empty-passphrase.pub.
On the replication master server etc/replication.config needs to be
adjusted as well:

 [remote "other-gerrit"]
     url = ssh://$replication_user@localhost:29419/${name}.git
     adminUrl = gerrit+ssh://$replication_user@localhost:29419/

After starting both instances creating a project in the master will
result in the same project created on the replication destination.
Of course Gerrit access rights still apply, therefore $replication_user
should have create project capability and push rights.

Change-Id: I677f7bd1164be259916c8cebdd4ddeee469402a3
Signed-off-by: Dariusz Luksza <dariusz@luksza.org>

5 files changed, 189 insertions, 27 deletions

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
new file mode 100644
index 0000000..98372d8
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
@@ -0,0 +1,129 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.ssh.SshAddressesModule;
+import com.google.inject.Inject;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URISyntaxException;
+import java.util.HashSet;
+import java.util.Set;
+import org.eclipse.jgit.transport.URIish;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GerritSshApi {
+  static int SSH_COMMAND_FAILED = -1;
+  private static final Logger log = LoggerFactory.getLogger(GerritSshApi.class);
+  private static String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+";
+
+  private final SshHelper sshHelper;
+
+  private final Set<URIish> withoutDeleteProjectPlugin = new HashSet<>();
+
+  @Inject
+  protected GerritSshApi(SshHelper sshHelper) {
+    this.sshHelper = sshHelper;
+  }
+
+  protected boolean createProject(URIish uri, Project.NameKey projectName, String head) {
+    OutputStream errStream = sshHelper.newErrorBufferStream();
+    String cmd = "gerrit create-project --branch " + head + " " + projectName.get();
+    try {
+      execute(uri, cmd, errStream);
+    } catch (IOException e) {
+      logError("creating", uri, errStream, cmd, e);
+      return false;
+    }
+    return true;
+  }
+
+  protected boolean deleteProject(URIish uri, Project.NameKey projectName) {
+    if (!withoutDeleteProjectPlugin.contains(uri)) {
+      OutputStream errStream = sshHelper.newErrorBufferStream();
+      String cmd = "deleteproject delete --yes-really-delete --force " + projectName.get();
+      int exitCode = -1;
+      try {
+        exitCode = execute(uri, cmd, errStream);
+      } catch (IOException e) {
+        logError("deleting", uri, errStream, cmd, e);
+        return false;
+      }
+      if (exitCode == 1) {
+        log.info(
+            "DeleteProject plugin is not installed on {}; will not try to forward this operation to that host");
+        withoutDeleteProjectPlugin.add(uri);
+        return true;
+      }
+    }
+    return true;
+  }
+
+  protected boolean updateHead(URIish uri, Project.NameKey projectName, String newHead) {
+    OutputStream errStream = sshHelper.newErrorBufferStream();
+    String cmd = "gerrit set-head " + projectName.get() + " --new-head " + newHead;
+    try {
+      execute(uri, cmd, errStream);
+    } catch (IOException e) {
+      log.error(
+          String.format(
+              "Error updating HEAD of remote repository at %s to %s:\n"
+                  + "  Exception: %s\n  Command: %s\n  Output: %s",
+              uri, newHead, e, cmd, errStream),
+          e);
+      return false;
+    }
+    return true;
+  }
+
+  private URIish toSshUri(URIish uri) throws URISyntaxException {
+    String uriStr = uri.toString();
+    if (uri.getHost() != null && uriStr.startsWith(GERRIT_ADMIN_PROTOCOL_PREFIX)) {
+      return new URIish(uriStr.substring(0, GERRIT_ADMIN_PROTOCOL_PREFIX.length()));
+    }
+    String rawPath = uri.getRawPath();
+    if (!rawPath.endsWith("/")) {
+      rawPath = rawPath + "/";
+    }
+    URIish sshUri = new URIish("ssh://" + rawPath);
+    if (sshUri.getPort() < 0) {
+      sshUri = sshUri.setPort(SshAddressesModule.DEFAULT_PORT);
+    }
+    return sshUri;
+  }
+
+  private int execute(URIish uri, String cmd, OutputStream errStream) throws IOException {
+    try {
+      URIish sshUri = toSshUri(uri);
+      return sshHelper.executeRemoteSsh(sshUri, cmd, errStream);
+    } catch (URISyntaxException e) {
+      log.error(String.format("Cannot convert %s to SSH uri", uri), e);
+    }
+    return SSH_COMMAND_FAILED;
+  }
+
+  public void logError(String msg, URIish uri, OutputStream errStream, String cmd, IOException e) {
+    log.error(
+        "Error {} remote repository at {}:\n  Exception: {}\n  Command: {}\n  Output: {}",
+        msg,
+        uri,
+        e,
+        cmd,
+        errStream,
+        e);
+  }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
index e472954..9b20485 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -349,7 +349,9 @@ class PushOne implements ProjectRunnable, CanceledWhileRunning {
       // does not exist.  In this case NoRemoteRepositoryException is not
       // raised.
       String msg = e.getMessage();
-      if (msg.contains("access denied") || msg.contains("no such repository")) {
+      if (msg.contains("access denied")
+          || msg.contains("no such repository")
+          || msg.contains("Git repository not found")) {
         createRepository();
       } else {
         repLog.error("Cannot replicate " + projectName + "; Remote repository error: " + msg);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
index 30f54ac..226c6fb 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
@@ -71,12 +71,14 @@ public class ReplicationQueue
   private final SshHelper sshHelper;
   private final DynamicItem<EventDispatcher> dispatcher;
   private final ReplicationConfig config;
+  private final GerritSshApi gerritAdmin;
   private volatile boolean running;
 
   @Inject
   ReplicationQueue(
       WorkQueue wq,
       SshHelper sh,
+      GerritSshApi ga,
       ReplicationConfig rc,
       DynamicItem<EventDispatcher> dis,
       ReplicationStateListener sl) {
@@ -85,6 +87,7 @@ public class ReplicationQueue
     dispatcher = dis;
     config = rc;
     stateLog = sl;
+    gerritAdmin = ga;
   }
 
   @Override
@@ -143,24 +146,25 @@ public class ReplicationQueue
 
   @Override
   public void onNewProjectCreated(NewProjectCreatedListener.Event event) {
-    for (URIish uri :
-        getURIs(new Project.NameKey(event.getProjectName()), FilterType.PROJECT_CREATION)) {
-      createProject(uri, event.getHeadName());
+    Project.NameKey projectName = new Project.NameKey(event.getProjectName());
+    for (URIish uri : getURIs(projectName, FilterType.PROJECT_CREATION)) {
+      createProject(uri, projectName, event.getHeadName());
     }
   }
 
   @Override
   public void onProjectDeleted(ProjectDeletedListener.Event event) {
-    for (URIish uri :
-        getURIs(new Project.NameKey(event.getProjectName()), FilterType.PROJECT_DELETION)) {
-      deleteProject(uri);
+    Project.NameKey projectName = new Project.NameKey(event.getProjectName());
+    for (URIish uri : getURIs(projectName, FilterType.PROJECT_DELETION)) {
+      deleteProject(uri, projectName);
     }
   }
 
   @Override
   public void onHeadUpdated(HeadUpdatedListener.Event event) {
-    for (URIish uri : getURIs(new Project.NameKey(event.getProjectName()), FilterType.ALL)) {
-      updateHead(uri, event.getNewHeadName());
+    Project.NameKey project = new Project.NameKey(event.getProjectName());
+    for (URIish uri : getURIs(project, FilterType.ALL)) {
+      updateHead(uri, project, event.getNewHeadName());
     }
   }
 
@@ -194,18 +198,20 @@ public class ReplicationQueue
           continue;
         }
 
-        String path = replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch());
-        if (path == null) {
-          repLog.warn(String.format("adminURL %s does not contain ${name}", uri));
-          continue;
-        }
+        if (!isGerrit(uri)) {
+          String path =
+              replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch());
+          if (path == null) {
+            repLog.warn(String.format("adminURL %s does not contain ${name}", uri));
+            continue;
+          }
 
-        uri = uri.setPath(path);
-        if (!isSSH(uri)) {
-          repLog.warn(String.format("adminURL '%s' is invalid: only SSH is supported", uri));
-          continue;
+          uri = uri.setPath(path);
+          if (!isSSH(uri)) {
+            repLog.warn(String.format("adminURL '%s' is invalid: only SSH is supported", uri));
+            continue;
+          }
         }
-
         uris.add(uri);
         adminURLUsed = true;
       }
@@ -222,13 +228,15 @@ public class ReplicationQueue
   public boolean createProject(Project.NameKey project, String head) {
     boolean success = true;
     for (URIish uri : getURIs(project, FilterType.PROJECT_CREATION)) {
-      success &= createProject(uri, head);
+      success &= createProject(uri, project, head);
     }
     return success;
   }
 
-  private boolean createProject(URIish replicateURI, String head) {
-    if (!replicateURI.isRemote()) {
+  private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) {
+    if (isGerrit(replicateURI)) {
+      gerritAdmin.createProject(replicateURI, projectName, head);
+    } else if (!replicateURI.isRemote()) {
       createLocally(replicateURI, head);
       repLog.info("Created local repository: " + replicateURI);
     } else if (isSSH(replicateURI)) {
@@ -281,8 +289,11 @@ public class ReplicationQueue
     }
   }
 
-  private void deleteProject(URIish replicateURI) {
-    if (!replicateURI.isRemote()) {
+  private void deleteProject(URIish replicateURI, Project.NameKey projectName) {
+    if (isGerrit(replicateURI)) {
+      gerritAdmin.deleteProject(replicateURI, projectName);
+      repLog.info("Deleted remote repository: " + replicateURI);
+    } else if (!replicateURI.isRemote()) {
       deleteLocally(replicateURI);
       repLog.info("Deleted local repository: " + replicateURI);
     } else if (isSSH(replicateURI)) {
@@ -342,8 +353,10 @@ public class ReplicationQueue
     }
   }
 
-  private void updateHead(URIish replicateURI, String newHead) {
-    if (!replicateURI.isRemote()) {
+  private void updateHead(URIish replicateURI, Project.NameKey projectName, String newHead) {
+    if (isGerrit(replicateURI)) {
+      gerritAdmin.updateHead(replicateURI, projectName, newHead);
+    } else if (!replicateURI.isRemote()) {
       updateHeadLocally(replicateURI, newHead);
     } else if (isSSH(replicateURI)) {
       updateHeadRemoteSsh(replicateURI, newHead);
@@ -401,4 +414,9 @@ public class ReplicationQueue
     }
     return false;
   }
+
+  private static boolean isGerrit(URIish uri) {
+    String scheme = uri.getScheme();
+    return scheme != null && scheme.toLowerCase().equals("gerrit+ssh");
+  }
 }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SshHelper.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SshHelper.java
index 56a9236..f32deab 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/SshHelper.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/SshHelper.java
@@ -35,7 +35,7 @@ class SshHelper {
     this.sshSessionFactoryProvider = sshSessionFactoryProvider;
   }
 
-  void executeRemoteSsh(URIish uri, String cmd, OutputStream errStream) throws IOException {
+  int executeRemoteSsh(URIish uri, String cmd, OutputStream errStream) throws IOException {
     RemoteSession ssh = connect(uri);
     Process proc = ssh.exec(cmd, 0);
     proc.getOutputStream().close();
@@ -51,6 +51,7 @@ class SshHelper {
       // Don't wait, drop out immediately.
     }
     ssh.disconnect();
+    return proc.exitValue();
   }
 
   OutputStream newErrorBufferStream() {
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 50664dd..bbcc51c 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -153,6 +153,18 @@ remote.NAME.adminUrl
 	local environment.  In that case, an alternative SSH url could
 	be specified to repository creation.
 
+	To enable replication to different Gerrit instance use `gerrit+ssh://`
+	as protocol name followed by hostname of another Gerrit server eg.
+
+	`gerrit+ssh://replica1.my.org/`
+
+	In this case replication will use Gerrit's SSH API to
+	create/remove projects and update repository HEAD references.
+
+	NOTE: In order to replicate project deletion, the
+	link:https://gerrit-review.googlesource.com/admin/projects/plugins/delete-project delete-project[delete-project]
+	plugin must be installed on the other Gerrit.
+
 remote.NAME.receivepack
 :	Path of the `git-receive-pack` executable on the remote
 	system, if using the SSH transport.