Merge "Expose jetty threadpool metrics via dropwizard" into stable-2.16

5 files changed, 143 insertions, 4 deletions

diff --git a/java/com/google/gerrit/metrics/dropwizard/BUILD b/java/com/google/gerrit/metrics/dropwizard/BUILD
index 4b3859f7a8..307980939f 100644
--- a/java/com/google/gerrit/metrics/dropwizard/BUILD
+++ b/java/com/google/gerrit/metrics/dropwizard/BUILD
@@ -8,6 +8,7 @@ java_library(
         "//java/com/google/gerrit/common:annotations",
         "//java/com/google/gerrit/extensions:api",
         "//java/com/google/gerrit/metrics",
+        "//java/com/google/gerrit/pgm/http/jetty",
         "//java/com/google/gerrit/server",
         "//lib:args4j",
         "//lib:guava",
diff --git a/java/com/google/gerrit/pgm/http/jetty/BUILD b/java/com/google/gerrit/pgm/http/jetty/BUILD
index ea3afe118c..43a8ed92b8 100644
--- a/java/com/google/gerrit/pgm/http/jetty/BUILD
+++ b/java/com/google/gerrit/pgm/http/jetty/BUILD
@@ -10,6 +10,7 @@ java_library(
         "//java/com/google/gerrit/httpd",
         "//java/com/google/gerrit/launcher",
         "//java/com/google/gerrit/lifecycle",
+        "//java/com/google/gerrit/metrics",
         "//java/com/google/gerrit/server",
         "//java/com/google/gerrit/server/util/time",
         "//java/com/google/gerrit/sshd",
diff --git a/java/com/google/gerrit/pgm/http/jetty/JettyMetrics.java b/java/com/google/gerrit/pgm/http/jetty/JettyMetrics.java
new file mode 100644
index 0000000000..92edf403d1
--- /dev/null
+++ b/java/com/google/gerrit/pgm/http/jetty/JettyMetrics.java
@@ -0,0 +1,91 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.gerrit.pgm.http.jetty;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.gerrit.metrics.CallbackMetric;
+import com.google.gerrit.metrics.CallbackMetric0;
+import com.google.gerrit.metrics.Description;
+import com.google.gerrit.metrics.MetricMaker;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+@Singleton
+public class JettyMetrics {
+
+  @Inject
+  JettyMetrics(JettyServer jetty, MetricMaker metrics) {
+    CallbackMetric0<Integer> minPoolSize =
+        metrics.newCallbackMetric(
+            "httpd/jetty/threadpool/min_pool_size",
+            Integer.class,
+            new Description("Minimum thread pool size").setGauge());
+    CallbackMetric0<Integer> maxPoolSize =
+        metrics.newCallbackMetric(
+            "httpd/jetty/threadpool/max_pool_size",
+            Integer.class,
+            new Description("Maximum thread pool size").setGauge());
+    CallbackMetric0<Integer> poolSize =
+        metrics.newCallbackMetric(
+            "httpd/jetty/threadpool/pool_size",
+            Integer.class,
+            new Description("Current thread pool size").setGauge());
+    CallbackMetric0<Integer> idleThreads =
+        metrics.newCallbackMetric(
+            "httpd/jetty/threadpool/idle_threads",
+            Integer.class,
+            new Description("Idle httpd threads").setGauge().setUnit("threads"));
+    CallbackMetric0<Integer> busyThreads =
+        metrics.newCallbackMetric(
+            "httpd/jetty/threadpool/active_threads",
+            Integer.class,
+            new Description("Active httpd threads").setGauge().setUnit("threads"));
+    CallbackMetric0<Integer> reservedThreads =
+        metrics.newCallbackMetric(
+            "httpd/jetty/threadpool/reserved_threads",
+            Integer.class,
+            new Description("Reserved httpd threads").setGauge().setUnit("threads"));
+    CallbackMetric0<Integer> queueSize =
+        metrics.newCallbackMetric(
+            "httpd/jetty/threadpool/queue_size",
+            Integer.class,
+            new Description("Thread pool queue size").setGauge().setUnit("requests"));
+    CallbackMetric0<Boolean> lowOnThreads =
+        metrics.newCallbackMetric(
+            "httpd/jetty/threadpool/is_low_on_threads",
+            Boolean.class,
+            new Description("Whether thread pool is low on threads").setGauge());
+    JettyServer.Metrics jettyMetrics = jetty.getMetrics();
+    metrics.newTrigger(
+        ImmutableSet.<CallbackMetric<?>>of(
+            idleThreads,
+            busyThreads,
+            reservedThreads,
+            minPoolSize,
+            maxPoolSize,
+            poolSize,
+            queueSize,
+            lowOnThreads),
+        () -> {
+          minPoolSize.set(jettyMetrics.getMinThreads());
+          maxPoolSize.set(jettyMetrics.getMaxThreads());
+          poolSize.set(jettyMetrics.getThreads());
+          idleThreads.set(jettyMetrics.getIdleThreads());
+          busyThreads.set(jettyMetrics.getBusyThreads());
+          reservedThreads.set(jettyMetrics.getReservedThreads());
+          queueSize.set(jettyMetrics.getQueueSize());
+          lowOnThreads.set(jettyMetrics.isLowOnThreads());
+        });
+  }
+}
diff --git a/java/com/google/gerrit/pgm/http/jetty/JettyModule.java b/java/com/google/gerrit/pgm/http/jetty/JettyModule.java
index c818276a5d..32a8b6d0f2 100644
--- a/java/com/google/gerrit/pgm/http/jetty/JettyModule.java
+++ b/java/com/google/gerrit/pgm/http/jetty/JettyModule.java
@@ -31,5 +31,6 @@ public class JettyModule extends LifecycleModule {
     bind(JettyServer.class);
     listener().to(JettyServer.Lifecycle.class);
     install(new FactoryModuleBuilder().build(HttpLogFactory.class));
+    bind(JettyMetrics.class);
   }
 }
diff --git a/java/com/google/gerrit/pgm/http/jetty/JettyServer.java b/java/com/google/gerrit/pgm/http/jetty/JettyServer.java
index 25a28a4cfd..8c7958c8c5 100644
--- a/java/com/google/gerrit/pgm/http/jetty/JettyServer.java
+++ b/java/com/google/gerrit/pgm/http/jetty/JettyServer.java
@@ -67,7 +67,6 @@ import org.eclipse.jetty.util.BlockingArrayQueue;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
-import org.eclipse.jetty.util.thread.ThreadPool;
 import org.eclipse.jgit.lib.Config;
 
 @Singleton
@@ -116,9 +115,49 @@ public class JettyServer {
     }
   }
 
+  static class Metrics {
+    private final QueuedThreadPool threadPool;
+
+    Metrics(QueuedThreadPool threadPool) {
+      this.threadPool = threadPool;
+    }
+
+    public int getIdleThreads() {
+      return threadPool.getIdleThreads();
+    }
+
+    public int getBusyThreads() {
+      return threadPool.getBusyThreads();
+    }
+
+    public int getReservedThreads() {
+      return threadPool.getReservedThreads();
+    }
+
+    public int getMinThreads() {
+      return threadPool.getMinThreads();
+    }
+
+    public int getMaxThreads() {
+      return threadPool.getMaxThreads();
+    }
+
+    public int getThreads() {
+      return threadPool.getThreads();
+    }
+
+    public int getQueueSize() {
+      return threadPool.getQueueSize();
+    }
+
+    public boolean isLowOnThreads() {
+      return threadPool.isLowOnThreads();
+    }
+  }
+
   private final SitePaths site;
   private final Server httpd;
-
+  private final Metrics metrics;
   private boolean reverseProxy;
 
   @Inject
@@ -130,8 +169,10 @@ public class JettyServer {
       HttpLogFactory httpLogFactory) {
     this.site = site;
 
-    httpd = new Server(threadPool(cfg, threadSettingsConfig));
+    QueuedThreadPool pool = threadPool(cfg, threadSettingsConfig);
+    httpd = new Server(pool);
     httpd.setConnectors(listen(httpd, cfg));
+    metrics = new Metrics(pool);
 
     Handler app = makeContext(env, cfg);
     if (cfg.getBoolean("httpd", "requestLog", !reverseProxy)) {
@@ -160,6 +201,10 @@ public class JettyServer {
     httpd.setStopAtShutdown(false);
   }
 
+  Metrics getMetrics() {
+    return metrics;
+  }
+
   private Connector[] listen(Server server, Config cfg) {
     // OpenID and certain web-based single-sign-on products can cause
     // some very long headers, especially in the Referer header. We
@@ -339,7 +384,7 @@ public class JettyServer {
     return site.resolve(path);
   }
 
-  private ThreadPool threadPool(Config cfg, ThreadSettingsConfig threadSettingsConfig) {
+  private QueuedThreadPool threadPool(Config cfg, ThreadSettingsConfig threadSettingsConfig) {
     int maxThreads = threadSettingsConfig.getHttpdMaxThreads();
     int minThreads = cfg.getInt("httpd", null, "minthreads", 5);
     int maxQueued = cfg.getInt("httpd", null, "maxqueued", 200);