Introduce repository size quota enforcer

This patch introduces "/repository:size" quota group that gets examined
when commits get pushed to the repository. It moves the responsibility of
quota check to Gerrit core so that implementation details don't have to
be exploited by quota plugin.
Pros:
* simplified implementation of quota plugin (only QuotaEnforcer needs
to be implemented)
* better compatibility in long term (quota plugin doesn't have to
recognise all the places when it is necessary to hook into in order
to deliver repository size quota functionality)

Change-Id: I481bec083be7f5b28bf69b9c8de119e74b32a095
Signed-off-by: Jacek Centkowski <jcentkowski@collab.net>

7 files changed, 300 insertions, 11 deletions

diff --git a/java/com/google/gerrit/acceptance/InProcessProtocol.java b/java/com/google/gerrit/acceptance/InProcessProtocol.java
index 7a79ce4929..30845a83d6 100644
--- a/java/com/google/gerrit/acceptance/InProcessProtocol.java
+++ b/java/com/google/gerrit/acceptance/InProcessProtocol.java
@@ -14,6 +14,10 @@
 
 package com.google.gerrit.acceptance;
 
+import static com.google.gerrit.server.git.receive.LazyPostReceiveHookChain.affectsSize;
+import static com.google.gerrit.server.quota.QuotaGroupDefinitions.REPOSITORY_SIZE_GROUP;
+
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.gerrit.acceptance.InProcessProtocol.Context;
 import com.google.gerrit.common.data.Capable;
@@ -40,6 +44,9 @@ import com.google.gerrit.server.permissions.ProjectPermission;
 import com.google.gerrit.server.plugincontext.PluginSetContext;
 import com.google.gerrit.server.project.ProjectCache;
 import com.google.gerrit.server.project.ProjectState;
+import com.google.gerrit.server.quota.QuotaBackend;
+import com.google.gerrit.server.quota.QuotaException;
+import com.google.gerrit.server.quota.QuotaResponse;
 import com.google.gerrit.server.util.RequestContext;
 import com.google.gerrit.server.util.RequestScopePropagator;
 import com.google.gerrit.server.util.ThreadLocalRequestContext;
@@ -263,6 +270,7 @@ class InProcessProtocol extends TestProtocol<Context> {
     private final DynamicSet<PostReceiveHook> postReceiveHooks;
     private final ThreadLocalRequestContext threadContext;
     private final PermissionBackend permissionBackend;
+    private final QuotaBackend quotaBackend;
 
     @Inject
     Receive(
@@ -273,7 +281,8 @@ class InProcessProtocol extends TestProtocol<Context> {
         PluginSetContext<ReceivePackInitializer> receivePackInitializers,
         DynamicSet<PostReceiveHook> postReceiveHooks,
         ThreadLocalRequestContext threadContext,
-        PermissionBackend permissionBackend) {
+        PermissionBackend permissionBackend,
+        QuotaBackend quotaBackend) {
       this.userProvider = userProvider;
       this.projectCache = projectCache;
       this.factory = factory;
@@ -282,6 +291,7 @@ class InProcessProtocol extends TestProtocol<Context> {
       this.postReceiveHooks = postReceiveHooks;
       this.threadContext = threadContext;
       this.permissionBackend = permissionBackend;
+      this.quotaBackend = quotaBackend;
     }
 
     @Override
@@ -321,10 +331,35 @@ class InProcessProtocol extends TestProtocol<Context> {
 
         receivePackInitializers.runEach(
             initializer -> initializer.init(projectState.getNameKey(), rp));
+        QuotaResponse.Aggregated availableTokens =
+            quotaBackend
+                .user(identifiedUser)
+                .project(req.project)
+                .availableTokens(REPOSITORY_SIZE_GROUP);
+        availableTokens.throwOnError();
+        availableTokens.availableTokens().ifPresent(v -> rp.setMaxObjectSizeLimit(v));
 
-        rp.setPostReceiveHook(PostReceiveHookChain.newChain(Lists.newArrayList(postReceiveHooks)));
+        ImmutableList<PostReceiveHook> hooks =
+            ImmutableList.<PostReceiveHook>builder()
+                .add(
+                    (pack, commands) -> {
+                      if (affectsSize(pack, commands)) {
+                        try {
+                          quotaBackend
+                              .user(identifiedUser)
+                              .project(req.project)
+                              .requestTokens(REPOSITORY_SIZE_GROUP, pack.getPackSize())
+                              .throwOnError();
+                        } catch (QuotaException e) {
+                          throw new RuntimeException(e);
+                        }
+                      }
+                    })
+                .addAll(postReceiveHooks)
+                .build();
+        rp.setPostReceiveHook(PostReceiveHookChain.newChain(hooks));
         return rp;
-      } catch (IOException | PermissionBackendException e) {
+      } catch (IOException | PermissionBackendException | QuotaException e) {
         throw new RuntimeException(e);
       }
     }
diff --git a/java/com/google/gerrit/server/git/receive/AsyncReceiveCommits.java b/java/com/google/gerrit/server/git/receive/AsyncReceiveCommits.java
index 66e66ca859..da2887f907 100644
--- a/java/com/google/gerrit/server/git/receive/AsyncReceiveCommits.java
+++ b/java/com/google/gerrit/server/git/receive/AsyncReceiveCommits.java
@@ -14,6 +14,7 @@
 
 package com.google.gerrit.server.git.receive;
 
+import static com.google.gerrit.server.quota.QuotaGroupDefinitions.REPOSITORY_SIZE_GROUP;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 
 import com.google.common.flogger.FluentLogger;
@@ -46,6 +47,9 @@ import com.google.gerrit.server.permissions.ProjectPermission;
 import com.google.gerrit.server.project.ContributorAgreementsChecker;
 import com.google.gerrit.server.project.ProjectState;
 import com.google.gerrit.server.query.change.InternalChangeQuery;
+import com.google.gerrit.server.quota.QuotaBackend;
+import com.google.gerrit.server.quota.QuotaException;
+import com.google.gerrit.server.quota.QuotaResponse;
 import com.google.gerrit.server.util.MagicBranch;
 import com.google.gerrit.server.util.RequestScopePropagator;
 import com.google.inject.Inject;
@@ -96,6 +100,7 @@ public class AsyncReceiveCommits implements PreReceiveHook {
   public static class Module extends PrivateModule {
     @Override
     public void configure() {
+      install(new FactoryModuleBuilder().build(LazyPostReceiveHookChain.Factory.class));
       install(new FactoryModuleBuilder().build(AsyncReceiveCommits.Factory.class));
       expose(AsyncReceiveCommits.Factory.class);
       // Don't expose the binding for ReceiveCommits.Factory. All callers should
@@ -253,9 +258,10 @@ public class AsyncReceiveCommits implements PreReceiveHook {
       RequestScopePropagator scopePropagator,
       ReceiveConfig receiveConfig,
       TransferConfig transferConfig,
-      Provider<LazyPostReceiveHookChain> lazyPostReceive,
+      LazyPostReceiveHookChain.Factory lazyPostReceive,
       ContributorAgreementsChecker contributorAgreements,
       Metrics metrics,
+      QuotaBackend quotaBackend,
       @Named(TIMEOUT_NAME) long timeoutMillis,
       @Assisted ProjectState projectState,
       @Assisted IdentifiedUser user,
@@ -284,7 +290,7 @@ public class AsyncReceiveCommits implements PreReceiveHook {
     receivePack.setRefFilter(new ReceiveRefFilter());
     receivePack.setAllowPushOptions(true);
     receivePack.setPreReceiveHook(this);
-    receivePack.setPostReceiveHook(lazyPostReceive.get());
+    receivePack.setPostReceiveHook(lazyPostReceive.create(user, projectName));
 
     // If the user lacks READ permission, some references may be filtered and hidden from view.
     // Check objects mentioned inside the incoming pack file are reachable from visible refs.
@@ -311,6 +317,17 @@ public class AsyncReceiveCommits implements PreReceiveHook {
         factory.create(
             projectState, user, receivePack, allRefsWatcher, messageSender, resultChangeIds);
     receiveCommits.init();
+    QuotaResponse.Aggregated availableTokens =
+        quotaBackend.user(user).project(projectName).availableTokens(REPOSITORY_SIZE_GROUP);
+    try {
+      availableTokens.throwOnError();
+    } catch (QuotaException e) {
+      logger.atWarning().withCause(e).log(
+          "Quota %s availableTokens request failed for project %s",
+          REPOSITORY_SIZE_GROUP, projectName);
+      throw new RuntimeException(e);
+    }
+    availableTokens.availableTokens().ifPresent(v -> receivePack.setMaxObjectSizeLimit(v));
   }
 
   /** Determine if the user can upload commits. */
diff --git a/java/com/google/gerrit/server/git/receive/LazyPostReceiveHookChain.java b/java/com/google/gerrit/server/git/receive/LazyPostReceiveHookChain.java
index 0f081bebb5..8e200eb67a 100644
--- a/java/com/google/gerrit/server/git/receive/LazyPostReceiveHookChain.java
+++ b/java/com/google/gerrit/server/git/receive/LazyPostReceiveHookChain.java
@@ -14,23 +14,78 @@
 
 package com.google.gerrit.server.git.receive;
 
+import static com.google.gerrit.server.quota.QuotaGroupDefinitions.REPOSITORY_SIZE_GROUP;
+
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.CurrentUser;
 import com.google.gerrit.server.plugincontext.PluginSetContext;
+import com.google.gerrit.server.quota.QuotaBackend;
+import com.google.gerrit.server.quota.QuotaResponse;
 import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
 import java.util.Collection;
 import org.eclipse.jgit.transport.PostReceiveHook;
 import org.eclipse.jgit.transport.ReceiveCommand;
 import org.eclipse.jgit.transport.ReceivePack;
 
-class LazyPostReceiveHookChain implements PostReceiveHook {
+/**
+ * Class is responsible for calling all registered post-receive hooks. In addition, in case when
+ * repository size quota is defined, it requests tokens (pack size) that were received. This is the
+ * final step of enforcing repository size quota that deducts token from available tokens.
+ */
+public class LazyPostReceiveHookChain implements PostReceiveHook {
+  interface Factory {
+    LazyPostReceiveHookChain create(CurrentUser user, Project.NameKey project);
+  }
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   private final PluginSetContext<PostReceiveHook> hooks;
+  private final QuotaBackend quotaBackend;
+  private final CurrentUser user;
+  private final Project.NameKey project;
 
   @Inject
-  LazyPostReceiveHookChain(PluginSetContext<PostReceiveHook> hooks) {
+  LazyPostReceiveHookChain(
+      PluginSetContext<PostReceiveHook> hooks,
+      QuotaBackend quotaBackend,
+      @Assisted CurrentUser user,
+      @Assisted Project.NameKey project) {
     this.hooks = hooks;
+    this.quotaBackend = quotaBackend;
+    this.user = user;
+    this.project = project;
   }
 
   @Override
   public void onPostReceive(ReceivePack rp, Collection<ReceiveCommand> commands) {
     hooks.runEach(h -> h.onPostReceive(rp, commands));
+    if (affectsSize(rp, commands)) {
+      QuotaResponse.Aggregated a =
+          quotaBackend
+              .user(user)
+              .project(project)
+              .requestTokens(REPOSITORY_SIZE_GROUP, rp.getPackSize());
+      if (a.hasError()) {
+        String msg =
+            String.format(
+                "%s request failed for project %s with [%s]",
+                REPOSITORY_SIZE_GROUP, project, a.errorMessage());
+        logger.atWarning().log(msg);
+        throw new RuntimeException(msg);
+      }
+    }
+  }
+
+  public static boolean affectsSize(ReceivePack rp, Collection<ReceiveCommand> commands) {
+    if (rp.getPackSize() > 0L) {
+      for (ReceiveCommand cmd : commands) {
+        if (cmd.getType() != ReceiveCommand.Type.DELETE) {
+          return true;
+        }
+      }
+    }
+    return false;
   }
 }
diff --git a/java/com/google/gerrit/server/quota/QuotaGroupDefinitions.java b/java/com/google/gerrit/server/quota/QuotaGroupDefinitions.java
new file mode 100644
index 0000000000..5110538866
--- /dev/null
+++ b/java/com/google/gerrit/server/quota/QuotaGroupDefinitions.java
@@ -0,0 +1,25 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.quota;
+
+public class QuotaGroupDefinitions {
+  /**
+   * Definition of repository size quota group. {@link QuotaEnforcer} implementations for repository
+   * size quota have to act on requests with this group name.
+   */
+  public static final String REPOSITORY_SIZE_GROUP = "/repository:size";
+
+  private QuotaGroupDefinitions() {}
+}
diff --git a/javatests/com/google/gerrit/acceptance/server/quota/DefaultQuotaBackendIT.java b/javatests/com/google/gerrit/acceptance/server/quota/DefaultQuotaBackendIT.java
index 3e902839b4..adc7807101 100644
--- a/javatests/com/google/gerrit/acceptance/server/quota/DefaultQuotaBackendIT.java
+++ b/javatests/com/google/gerrit/acceptance/server/quota/DefaultQuotaBackendIT.java
@@ -21,6 +21,8 @@ import static org.easymock.EasyMock.resetToStrict;
 
 import com.google.gerrit.acceptance.AbstractDaemonTest;
 import com.google.gerrit.extensions.annotations.Exports;
+import com.google.gerrit.extensions.common.ChangeInfo;
+import com.google.gerrit.extensions.common.ChangeInput;
 import com.google.gerrit.extensions.config.FactoryModule;
 import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.server.IdentifiedUser;
@@ -92,7 +94,7 @@ public class DefaultQuotaBackendIT extends AbstractDaemonTest {
 
   @Test
   public void requestTokenForUserAndChange() throws Exception {
-    Change.Id changeId = createChange().getChange().getId();
+    Change.Id changeId = retrieveChangeId();
     QuotaRequestContext ctx =
         QuotaRequestContext.builder()
             .user(identifiedAdmin)
@@ -148,7 +150,7 @@ public class DefaultQuotaBackendIT extends AbstractDaemonTest {
 
   @Test
   public void availableTokensForUserAndChange() throws Exception {
-    Change.Id changeId = createChange().getChange().getId();
+    Change.Id changeId = retrieveChangeId();
     QuotaRequestContext ctx =
         QuotaRequestContext.builder()
             .user(identifiedAdmin)
@@ -224,6 +226,13 @@ public class DefaultQuotaBackendIT extends AbstractDaemonTest {
     quotaBackend.user(identifiedAdmin).availableTokens("testGroup");
   }
 
+  private Change.Id retrieveChangeId() throws Exception {
+    // use REST API so that repository size quota doesn't have to be stubbed
+    ChangeInfo changeInfo =
+        gApi.changes().create(new ChangeInput(project.get(), "master", "test")).get();
+    return new Change.Id(changeInfo._number);
+  }
+
   private static QuotaResponse.Aggregated singletonAggregation(QuotaResponse response) {
     return QuotaResponse.Aggregated.create(Collections.singleton(response));
   }
diff --git a/javatests/com/google/gerrit/acceptance/server/quota/RepositorySizeQuotaIT.java b/javatests/com/google/gerrit/acceptance/server/quota/RepositorySizeQuotaIT.java
new file mode 100644
index 0000000000..0814230e30
--- /dev/null
+++ b/javatests/com/google/gerrit/acceptance/server/quota/RepositorySizeQuotaIT.java
@@ -0,0 +1,140 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.acceptance.server.quota;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assert_;
+import static com.google.gerrit.server.quota.QuotaGroupDefinitions.REPOSITORY_SIZE_GROUP;
+import static com.google.gerrit.server.quota.QuotaResponse.ok;
+import static org.easymock.EasyMock.anyLong;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.resetToStrict;
+import static org.easymock.EasyMock.verify;
+
+import com.google.gerrit.acceptance.AbstractDaemonTest;
+import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.extensions.config.FactoryModule;
+import com.google.gerrit.server.CurrentUser;
+import com.google.gerrit.server.quota.QuotaBackend;
+import com.google.gerrit.server.quota.QuotaResponse;
+import com.google.inject.Module;
+import java.util.Collections;
+import org.easymock.EasyMock;
+import org.eclipse.jgit.api.errors.TooLargeObjectInPackException;
+import org.eclipse.jgit.api.errors.TransportException;
+import org.junit.Before;
+import org.junit.Test;
+
+@UseLocalDisk
+public class RepositorySizeQuotaIT extends AbstractDaemonTest {
+  private static final QuotaBackend.WithResource quotaBackendWithResource =
+      EasyMock.createStrictMock(QuotaBackend.WithResource.class);
+  private static final QuotaBackend.WithUser quotaBackendWithUser =
+      EasyMock.createStrictMock(QuotaBackend.WithUser.class);
+
+  @Override
+  public Module createModule() {
+    return new FactoryModule() {
+      @Override
+      public void configure() {
+        bind(QuotaBackend.class)
+            .toInstance(
+                new QuotaBackend() {
+                  @Override
+                  public WithUser currentUser() {
+                    return quotaBackendWithUser;
+                  }
+
+                  @Override
+                  public WithUser user(CurrentUser user) {
+                    return quotaBackendWithUser;
+                  }
+                });
+      }
+    };
+  }
+
+  @Before
+  public void setUp() {
+    resetToStrict(quotaBackendWithResource);
+    resetToStrict(quotaBackendWithUser);
+  }
+
+  @Test
+  public void pushWithAvailableTokens() throws Exception {
+    expect(quotaBackendWithResource.availableTokens(REPOSITORY_SIZE_GROUP))
+        .andReturn(singletonAggregation(ok(276L)))
+        .times(2);
+    expect(quotaBackendWithResource.requestTokens(eq(REPOSITORY_SIZE_GROUP), anyLong()))
+        .andReturn(singletonAggregation(ok()));
+    expect(quotaBackendWithUser.project(project)).andReturn(quotaBackendWithResource).anyTimes();
+    replay(quotaBackendWithResource);
+    replay(quotaBackendWithUser);
+    pushCommit();
+    verify(quotaBackendWithUser);
+    verify(quotaBackendWithResource);
+  }
+
+  @Test
+  public void pushWithNotSufficientTokens() throws Exception {
+    long availableTokens = 1L;
+    expect(quotaBackendWithResource.availableTokens(REPOSITORY_SIZE_GROUP))
+        .andReturn(singletonAggregation(ok(availableTokens)))
+        .anyTimes();
+    expect(quotaBackendWithUser.project(project)).andReturn(quotaBackendWithResource).anyTimes();
+    replay(quotaBackendWithResource);
+    replay(quotaBackendWithUser);
+    try {
+      pushCommit();
+      assert_().fail("expected TooLargeObjectInPackException");
+    } catch (TooLargeObjectInPackException e) {
+      String msg = e.getMessage();
+      assertThat(msg).contains("Object too large");
+      assertThat(msg)
+          .contains(String.format("Max object size limit is %d bytes.", availableTokens));
+    }
+    verify(quotaBackendWithUser);
+    verify(quotaBackendWithResource);
+  }
+
+  @Test
+  public void errorGettingAvailableTokens() throws Exception {
+    String msg = "quota error";
+    expect(quotaBackendWithResource.availableTokens(REPOSITORY_SIZE_GROUP))
+        .andReturn(singletonAggregation(QuotaResponse.error(msg)))
+        .anyTimes();
+    expect(quotaBackendWithUser.project(project)).andReturn(quotaBackendWithResource).anyTimes();
+    replay(quotaBackendWithResource);
+    replay(quotaBackendWithUser);
+    try {
+      pushCommit();
+      assert_().fail("expected TransportException");
+    } catch (TransportException e) {
+      // TransportException has not much info about the cause
+    }
+    verify(quotaBackendWithUser);
+    verify(quotaBackendWithResource);
+  }
+
+  private void pushCommit() throws Exception {
+    createCommitAndPush(testRepo, "refs/heads/master", "test 01", "file.test", "some content");
+  }
+
+  private static QuotaResponse.Aggregated singletonAggregation(QuotaResponse response) {
+    return QuotaResponse.Aggregated.create(Collections.singleton(response));
+  }
+}
diff --git a/javatests/com/google/gerrit/acceptance/server/quota/RestApiQuotaIT.java b/javatests/com/google/gerrit/acceptance/server/quota/RestApiQuotaIT.java
index a07569044d..dc082feda9 100644
--- a/javatests/com/google/gerrit/acceptance/server/quota/RestApiQuotaIT.java
+++ b/javatests/com/google/gerrit/acceptance/server/quota/RestApiQuotaIT.java
@@ -20,6 +20,7 @@ import static org.easymock.EasyMock.reset;
 import static org.easymock.EasyMock.verify;
 
 import com.google.gerrit.acceptance.AbstractDaemonTest;
+import com.google.gerrit.extensions.common.ChangeInfo;
 import com.google.gerrit.extensions.common.ChangeInput;
 import com.google.gerrit.extensions.config.FactoryModule;
 import com.google.gerrit.reviewdb.client.Change;
@@ -68,7 +69,7 @@ public class RestApiQuotaIT extends AbstractDaemonTest {
 
   @Test
   public void changeDetail() throws Exception {
-    Change.Id changeId = createChange().getChange().getId();
+    Change.Id changeId = retrieveChangeId();
     expect(quotaBackendWithResource.requestToken("/restapi/changes/detail:GET"))
         .andReturn(singletonAggregation(QuotaResponse.ok()));
     replay(quotaBackendWithResource);
@@ -81,7 +82,7 @@ public class RestApiQuotaIT extends AbstractDaemonTest {
 
   @Test
   public void revisionDetail() throws Exception {
-    Change.Id changeId = createChange().getChange().getId();
+    Change.Id changeId = retrieveChangeId();
     expect(quotaBackendWithResource.requestToken("/restapi/changes/revisions/actions:GET"))
         .andReturn(singletonAggregation(QuotaResponse.ok()));
     replay(quotaBackendWithResource);
@@ -130,6 +131,13 @@ public class RestApiQuotaIT extends AbstractDaemonTest {
     adminRestSession.get("/config/server/version").assertStatus(429);
   }
 
+  private Change.Id retrieveChangeId() throws Exception {
+    // use REST API so that repository size quota doesn't have to be stubbed
+    ChangeInfo changeInfo =
+        gApi.changes().create(new ChangeInput(project.get(), "master", "test")).get();
+    return new Change.Id(changeInfo._number);
+  }
+
   private static QuotaResponse.Aggregated singletonAggregation(QuotaResponse response) {
     return QuotaResponse.Aggregated.create(Collections.singleton(response));
   }