diff options
11 files changed, 393 insertions, 6 deletions
diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt index dbe390957e..8ad5cef739 100644 --- a/Documentation/config-gerrit.txt +++ b/Documentation/config-gerrit.txt @@ -4008,6 +4008,46 @@ If set to true, log files are rotated daily at midnight (GMT). + Defaults to true. +[[metrics]] +=== Section metrics + +[[metrics.reservoir]]metrics.reservoir:: ++ +The type of data reservoir used by the metrics system to calculate the percentile +values for timers and histograms. +It can be set to one of the following values: ++ +* ExponentiallyDecaying: An exponentially-decaying random reservoir based on + Cormode et al's forward-decaying priority reservoir sampling method to produce + a statistically representative sampling reservoir, exponentially biased towards + newer entries. +* SlidingTimeWindowArray: A sliding window that stores only the measurements made + in the last window using chunks of 512 samples. +* SlidingTimeWindow: A sliding window that stores only the measurements made in + the last window using a skip list. +* SlidingWindow: A sliding window that stores only the last measurements. +* Uniform: A random sampling reservoir that uses Vitter's Algorithm R to produce + a statistically representative sample. ++ +Defaults to ExponentiallyDecaying. + +[[metrics.ExponentiallyDecaying.alpha]]metrics.ExponentiallyDecaying.alpha:: ++ +The exponential decay factor; the higher this is, the more biased the reservoir +will be towards newer values. + +[[metrics.reservoirType.size]]metrics.<reservoirType>.size:: ++ +The number of samples to keep in the reservoir. Applies to all reservoir types +except the sliding time-based ones. ++ +Defaults to 1028. + +[[metrics.reservoirType.window]]metrics.<reservoirType>.window:: ++ +The window of time for keeping data in the reservoir. It only applies to sliding +time-based reservoir types. + [[mimetype]] === Section mimetype diff --git a/java/com/google/gerrit/acceptance/InMemoryTestingDatabaseModule.java b/java/com/google/gerrit/acceptance/InMemoryTestingDatabaseModule.java index de9a43d6d6..c2b21fbb11 100644 --- a/java/com/google/gerrit/acceptance/InMemoryTestingDatabaseModule.java +++ b/java/com/google/gerrit/acceptance/InMemoryTestingDatabaseModule.java @@ -21,7 +21,9 @@ import com.google.gerrit.exceptions.StorageException; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.lifecycle.LifecycleModule; import com.google.gerrit.metrics.MetricMaker; +import com.google.gerrit.metrics.MetricsReservoirConfig; import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.MetricsReservoirConfigImpl; import com.google.gerrit.server.config.SitePath; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.config.TrackingFooters; @@ -32,6 +34,7 @@ import com.google.gerrit.server.schema.SchemaModule; import com.google.gerrit.testing.InMemoryRepositoryManager; import com.google.inject.Inject; import com.google.inject.ProvisionException; +import com.google.inject.Scopes; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -63,6 +66,7 @@ class InMemoryTestingDatabaseModule extends LifecycleModule { bind(InMemoryRepositoryManager.class).in(SINGLETON); } + bind(MetricsReservoirConfig.class).to(MetricsReservoirConfigImpl.class).in(Scopes.SINGLETON); bind(MetricMaker.class).to(TestMetricMaker.class); listener().to(CreateSchema.class); diff --git a/java/com/google/gerrit/metrics/MetricsReservoirConfig.java b/java/com/google/gerrit/metrics/MetricsReservoirConfig.java new file mode 100644 index 0000000000..455dfc0829 --- /dev/null +++ b/java/com/google/gerrit/metrics/MetricsReservoirConfig.java @@ -0,0 +1,33 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.metrics; + +import java.time.Duration; + +/** Configuration of the Metrics' reservoir type and size. */ +public interface MetricsReservoirConfig { + + /** @return the reservoir type. */ + ReservoirType reservoirType(); + + /** @return the reservoir window duration. */ + Duration reservoirWindow(); + + /** @return the number of samples that the reservoir can contain */ + int reservoirSize(); + + /** @return the alpha parameter of the ExponentiallyDecaying reservoir */ + double reservoirAlpha(); +} diff --git a/java/com/google/gerrit/metrics/ReservoirType.java b/java/com/google/gerrit/metrics/ReservoirType.java new file mode 100644 index 0000000000..fe8975205f --- /dev/null +++ b/java/com/google/gerrit/metrics/ReservoirType.java @@ -0,0 +1,24 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.metrics; + +/** Type of reservoir for collecting metrics into. */ +public enum ReservoirType { + ExponentiallyDecaying, + SlidingTimeWindowArray, + SlidingTimeWindow, + SlidingWindow, + Uniform; +} diff --git a/java/com/google/gerrit/metrics/dropwizard/DropWizardMetricMaker.java b/java/com/google/gerrit/metrics/dropwizard/DropWizardMetricMaker.java index fcba0eeb3c..32be18d981 100644 --- a/java/com/google/gerrit/metrics/dropwizard/DropWizardMetricMaker.java +++ b/java/com/google/gerrit/metrics/dropwizard/DropWizardMetricMaker.java @@ -18,8 +18,10 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.gerrit.metrics.dropwizard.MetricResource.METRIC_KIND; import static com.google.gerrit.server.config.ConfigResource.CONFIG_KIND; +import com.codahale.metrics.Histogram; import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -41,6 +43,7 @@ import com.google.gerrit.metrics.Histogram1; import com.google.gerrit.metrics.Histogram2; import com.google.gerrit.metrics.Histogram3; import com.google.gerrit.metrics.MetricMaker; +import com.google.gerrit.metrics.MetricsReservoirConfig; import com.google.gerrit.metrics.Timer0; import com.google.gerrit.metrics.Timer1; import com.google.gerrit.metrics.Timer2; @@ -48,10 +51,12 @@ import com.google.gerrit.metrics.Timer3; import com.google.gerrit.metrics.proc.JGitMetricModule; import com.google.gerrit.metrics.proc.ProcMetricModule; import com.google.gerrit.server.cache.CacheMetrics; +import com.google.gerrit.server.config.MetricsReservoirConfigImpl; import com.google.inject.Inject; import com.google.inject.Scopes; import com.google.inject.Singleton; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -65,8 +70,25 @@ import java.util.regex.Pattern; @Singleton public class DropWizardMetricMaker extends MetricMaker { public static class ApiModule extends RestApiModule { + private final Optional<MetricsReservoirConfig> metricsReservoirConfig; + + public ApiModule(MetricsReservoirConfig metricsReservoirConfig) { + this.metricsReservoirConfig = Optional.of(metricsReservoirConfig); + } + + public ApiModule() { + this.metricsReservoirConfig = Optional.empty(); + } + @Override protected void configure() { + if (metricsReservoirConfig.isPresent()) { + bind(MetricsReservoirConfig.class).toInstance(metricsReservoirConfig.get()); + } else { + bind(MetricsReservoirConfig.class) + .to(MetricsReservoirConfigImpl.class) + .in(Scopes.SINGLETON); + } bind(MetricRegistry.class).in(Scopes.SINGLETON); bind(DropWizardMetricMaker.class).in(Scopes.SINGLETON); bind(MetricMaker.class).to(DropWizardMetricMaker.class); @@ -89,12 +111,14 @@ public class DropWizardMetricMaker extends MetricMaker { private final MetricRegistry registry; private final Map<String, BucketedMetric> bucketed; private final Map<String, ImmutableMap<String, String>> descriptions; + private final MetricsReservoirConfig reservoirConfig; @Inject - DropWizardMetricMaker(MetricRegistry registry) { + DropWizardMetricMaker(MetricRegistry registry, MetricsReservoirConfig reservoirConfig) { this.registry = registry; this.bucketed = new ConcurrentHashMap<>(); this.descriptions = new ConcurrentHashMap<>(); + this.reservoirConfig = reservoirConfig; } Iterable<String> getMetricNames() { @@ -222,7 +246,9 @@ public class DropWizardMetricMaker extends MetricMaker { } TimerImpl newTimerImpl(String name) { - return new TimerImpl(name, registry.timer(name)); + return new TimerImpl( + name, + registry.timer(name, () -> new Timer(DropWizardReservoirProvider.get(reservoirConfig)))); } @Override @@ -271,7 +297,10 @@ public class DropWizardMetricMaker extends MetricMaker { } HistogramImpl newHistogramImpl(String name) { - return new HistogramImpl(name, registry.histogram(name)); + return new HistogramImpl( + name, + registry.histogram( + name, () -> new Histogram(DropWizardReservoirProvider.get(reservoirConfig)))); } @Override diff --git a/java/com/google/gerrit/metrics/dropwizard/DropWizardReservoirProvider.java b/java/com/google/gerrit/metrics/dropwizard/DropWizardReservoirProvider.java new file mode 100644 index 0000000000..30890689cb --- /dev/null +++ b/java/com/google/gerrit/metrics/dropwizard/DropWizardReservoirProvider.java @@ -0,0 +1,52 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.metrics.dropwizard; + +import com.codahale.metrics.ExponentiallyDecayingReservoir; +import com.codahale.metrics.Reservoir; +import com.codahale.metrics.SlidingTimeWindowArrayReservoir; +import com.codahale.metrics.SlidingTimeWindowReservoir; +import com.codahale.metrics.SlidingWindowReservoir; +import com.codahale.metrics.UniformReservoir; +import com.google.gerrit.metrics.MetricsReservoirConfig; +import com.google.gerrit.metrics.ReservoirType; +import java.util.concurrent.TimeUnit; + +class DropWizardReservoirProvider { + + private DropWizardReservoirProvider() {} + + static Reservoir get(MetricsReservoirConfig config) { + ReservoirType reservoirType = config.reservoirType(); + switch (reservoirType) { + case ExponentiallyDecaying: + return new ExponentiallyDecayingReservoir(config.reservoirSize(), config.reservoirAlpha()); + case SlidingTimeWindowArray: + return new SlidingTimeWindowArrayReservoir( + config.reservoirWindow().toMillis(), TimeUnit.MILLISECONDS); + case SlidingTimeWindow: + return new SlidingTimeWindowReservoir( + config.reservoirWindow().toMillis(), TimeUnit.MILLISECONDS); + case SlidingWindow: + return new SlidingWindowReservoir(config.reservoirSize()); + case Uniform: + return new UniformReservoir(config.reservoirSize()); + + default: + throw new IllegalArgumentException( + "Unsupported metrics reservoir type " + reservoirType.name()); + } + } +} diff --git a/java/com/google/gerrit/server/config/MetricsReservoirConfigImpl.java b/java/com/google/gerrit/server/config/MetricsReservoirConfigImpl.java new file mode 100644 index 0000000000..ac3c53acf7 --- /dev/null +++ b/java/com/google/gerrit/server/config/MetricsReservoirConfigImpl.java @@ -0,0 +1,96 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.config; + +import com.google.gerrit.metrics.MetricsReservoirConfig; +import com.google.gerrit.metrics.ReservoirType; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.eclipse.jgit.lib.Config; + +/** Define metrics reservoir settings based on gerrit.config */ +@Singleton +public class MetricsReservoirConfigImpl implements MetricsReservoirConfig { + private static final double RESERVOIR_ALPHA_DEFAULT = 0.015; + private static final int METRICS_RESERVOIR_SIZE_DEFAULT = 1028; + private static final long METRICS_RESERVOIR_WINDOW_MSEC_DEFAULT = 60000L; + private static final String METRICS_SECTION = "metrics"; + private static final String METRICS_RESERVOIR = "reservoir"; + + private final ReservoirType reservoirType; + + private final Duration reservoirWindow; + private final int reservoirSize; + private final double reservoirAlpha; + + @Inject + MetricsReservoirConfigImpl(@GerritServerConfig Config gerritConfig) { + this.reservoirType = + gerritConfig.getEnum( + METRICS_SECTION, null, METRICS_RESERVOIR, ReservoirType.ExponentiallyDecaying); + + reservoirWindow = + Duration.ofMillis( + ConfigUtil.getTimeUnit( + gerritConfig, + METRICS_SECTION, + reservoirType.name(), + "window", + METRICS_RESERVOIR_WINDOW_MSEC_DEFAULT, + TimeUnit.MILLISECONDS)); + reservoirSize = + gerritConfig.getInt( + METRICS_SECTION, reservoirType.name(), "size", METRICS_RESERVOIR_SIZE_DEFAULT); + reservoirAlpha = + Optional.ofNullable(gerritConfig.getString(METRICS_SECTION, reservoirType.name(), "alpha")) + .map(Double::parseDouble) + .orElse(RESERVOIR_ALPHA_DEFAULT); + } + + /* (non-Javadoc) + * @see com.google.gerrit.server.config.MetricsConfig#reservoirType() + */ + @Override + public ReservoirType reservoirType() { + return reservoirType; + } + + /* (non-Javadoc) + * @see com.google.gerrit.server.config.MetricsConfig#reservoirWindow() + */ + @Override + public Duration reservoirWindow() { + return reservoirWindow; + } + + /* (non-Javadoc) + * @see com.google.gerrit.server.config.MetricsConfig#reservoirSize() + */ + @Override + public int reservoirSize() { + return reservoirSize; + } + + /* (non-Javadoc) + * @see com.google.gerrit.server.config.MetricsConfig#reservoirAlpha() + */ + @Override + public double reservoirAlpha() { + return reservoirAlpha; + } +} diff --git a/javatests/com/google/gerrit/metrics/dropwizard/BUILD b/javatests/com/google/gerrit/metrics/dropwizard/BUILD index 98d12b2e7b..e236f30ce5 100644 --- a/javatests/com/google/gerrit/metrics/dropwizard/BUILD +++ b/javatests/com/google/gerrit/metrics/dropwizard/BUILD @@ -6,7 +6,10 @@ junit_tests( tags = ["metrics"], visibility = ["//visibility:public"], deps = [ + "//java/com/google/gerrit/metrics", "//java/com/google/gerrit/metrics/dropwizard", + "//lib/mockito", "//lib/truth", + "@dropwizard-core//jar", ], ) diff --git a/javatests/com/google/gerrit/metrics/dropwizard/DropWizardMetricMakerTest.java b/javatests/com/google/gerrit/metrics/dropwizard/DropWizardMetricMakerTest.java index 9b21bf6c01..5777779513 100644 --- a/javatests/com/google/gerrit/metrics/dropwizard/DropWizardMetricMakerTest.java +++ b/javatests/com/google/gerrit/metrics/dropwizard/DropWizardMetricMakerTest.java @@ -15,12 +15,33 @@ package com.google.gerrit.metrics.dropwizard; import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import com.codahale.metrics.MetricRegistry; +import com.google.gerrit.metrics.Description; +import com.google.gerrit.metrics.MetricsReservoirConfig; +import com.google.gerrit.metrics.ReservoirType; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +@RunWith(MockitoJUnitRunner.class) public class DropWizardMetricMakerTest { - DropWizardMetricMaker metrics = - new DropWizardMetricMaker(null /* MetricRegistry unused in tests */); + + @Mock MetricsReservoirConfig reservoirConfigMock; + + MetricRegistry registry; + + DropWizardMetricMaker metrics; + + @Before + public void setupMocks() { + registry = new MetricRegistry(); + metrics = new DropWizardMetricMaker(registry, reservoirConfigMock); + } @Test public void shouldSanitizeUnwantedChars() throws Exception { @@ -41,4 +62,15 @@ public class DropWizardMetricMakerTest { assertThat(metrics.sanitizeMetricName("metric//")).isEqualTo("metric"); assertThat(metrics.sanitizeMetricName("metric/submetric/")).isEqualTo("metric/submetric"); } + + @Test + public void shouldRequestForReservoirForNewTimer() throws Exception { + when(reservoirConfigMock.reservoirType()).thenReturn(ReservoirType.ExponentiallyDecaying); + + metrics.newTimer( + "foo", + new Description("foo description").setCumulative().setUnit(Description.Units.MILLISECONDS)); + + verify(reservoirConfigMock).reservoirType(); + } } diff --git a/javatests/com/google/gerrit/metrics/dropwizard/DropWizardReservoirProviderTest.java b/javatests/com/google/gerrit/metrics/dropwizard/DropWizardReservoirProviderTest.java new file mode 100644 index 0000000000..6402b53ad4 --- /dev/null +++ b/javatests/com/google/gerrit/metrics/dropwizard/DropWizardReservoirProviderTest.java @@ -0,0 +1,64 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.metrics.dropwizard; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.when; + +import com.codahale.metrics.ExponentiallyDecayingReservoir; +import com.codahale.metrics.SlidingTimeWindowArrayReservoir; +import com.codahale.metrics.SlidingTimeWindowReservoir; +import com.codahale.metrics.SlidingWindowReservoir; +import com.codahale.metrics.UniformReservoir; +import com.google.gerrit.metrics.MetricsReservoirConfig; +import com.google.gerrit.metrics.ReservoirType; +import java.time.Duration; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class DropWizardReservoirProviderTest { + private static final int SLIDING_WINDOW_INTERVAL = 1; + private static final int SLIDING_WINDOW_SIZE = 256; + + @Mock private MetricsReservoirConfig configMock; + + @Test + public void shouldInstantiateReservoirProviderBasedOnMetricsConfig() { + when(configMock.reservoirType()).thenReturn(ReservoirType.ExponentiallyDecaying); + assertThat(DropWizardReservoirProvider.get(configMock)) + .isInstanceOf(ExponentiallyDecayingReservoir.class); + + when(configMock.reservoirType()).thenReturn(ReservoirType.SlidingTimeWindow); + when(configMock.reservoirWindow()).thenReturn(Duration.ofMinutes(1)); + assertThat(DropWizardReservoirProvider.get(configMock)) + .isInstanceOf(SlidingTimeWindowReservoir.class); + + when(configMock.reservoirType()).thenReturn(ReservoirType.SlidingTimeWindowArray); + when(configMock.reservoirWindow()).thenReturn(Duration.ofMinutes(SLIDING_WINDOW_INTERVAL)); + assertThat(DropWizardReservoirProvider.get(configMock)) + .isInstanceOf(SlidingTimeWindowArrayReservoir.class); + + when(configMock.reservoirType()).thenReturn(ReservoirType.SlidingWindow); + when(configMock.reservoirSize()).thenReturn(SLIDING_WINDOW_SIZE); + assertThat(DropWizardReservoirProvider.get(configMock)) + .isInstanceOf(SlidingWindowReservoir.class); + + when(configMock.reservoirType()).thenReturn(ReservoirType.Uniform); + assertThat(DropWizardReservoirProvider.get(configMock)).isInstanceOf(UniformReservoir.class); + } +} diff --git a/javatests/com/google/gerrit/metrics/proc/ProcMetricModuleTest.java b/javatests/com/google/gerrit/metrics/proc/ProcMetricModuleTest.java index 33919e7ad0..ea89ae9d92 100644 --- a/javatests/com/google/gerrit/metrics/proc/ProcMetricModuleTest.java +++ b/javatests/com/google/gerrit/metrics/proc/ProcMetricModuleTest.java @@ -17,6 +17,7 @@ package com.google.gerrit.metrics.proc; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static com.google.gerrit.testing.GerritJUnit.assertThrows; +import static org.mockito.Mockito.mock; import com.codahale.metrics.Counter; import com.codahale.metrics.Gauge; @@ -31,7 +32,9 @@ import com.google.gerrit.metrics.Description; import com.google.gerrit.metrics.Description.FieldOrdering; import com.google.gerrit.metrics.Field; import com.google.gerrit.metrics.MetricMaker; +import com.google.gerrit.metrics.MetricsReservoirConfig; import com.google.gerrit.metrics.dropwizard.DropWizardMetricMaker; +import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; @@ -179,7 +182,14 @@ public class ProcMetricModuleTest { @Before public void setup() { - Injector injector = Guice.createInjector(new DropWizardMetricMaker.ApiModule()); + Injector injector = + Guice.createInjector( + new AbstractModule() { + @Override + protected void configure() { + install(new DropWizardMetricMaker.ApiModule(mock(MetricsReservoirConfig.class))); + } + }); LifecycleManager mgr = new LifecycleManager(); mgr.add(injector); |