context) {
+ numberOfExecutions.incrementAndGet();
+
+ var statusPatch = new PerformanceCustomResource();
+ statusPatch.setMetadata(
+ new ObjectMetaBuilder()
+ .withName(resource.getMetadata().getName())
+ .withNamespace(resource.getMetadata().getNamespace())
+ .build());
+ var status = new PerformanceCustomResourceStatus();
+ status.setObservedGeneration(resource.getMetadata().getGeneration());
+ status.setObservedValue(resource.getSpec().getValue());
+ statusPatch.setStatus(status);
+
+ return UpdateControl.patchStatus(statusPatch);
+ }
+
+ public int getNumberOfExecutions() {
+ return numberOfExecutions.get();
+ }
+}
diff --git a/performance-tests/e2e/src/test/java/io/javaoperatorsdk/operator/performance/ReconciliationThroughputE2E.java b/performance-tests/e2e/src/test/java/io/javaoperatorsdk/operator/performance/ReconciliationThroughputE2E.java
new file mode 100644
index 0000000000..12ed931b1d
--- /dev/null
+++ b/performance-tests/e2e/src/test/java/io/javaoperatorsdk/operator/performance/ReconciliationThroughputE2E.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance;
+
+import java.time.Duration;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
+import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension;
+import io.javaoperatorsdk.operator.performance.results.PerformanceTest;
+import io.javaoperatorsdk.operator.performance.results.PerformanceTestResults;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
+/**
+ * Measures how fast a locally run operator reconciles a batch of custom resources against a real
+ * cluster. The reconciler itself does close to nothing, so what is measured is the SDK event
+ * processing plus the API server round trips.
+ *
+ * The scenario is tunable with system properties:
+ *
+ *
+ * - {@code performance.resourceCount} - number of custom resources to create (default 100)
+ *
- {@code performance.timeoutSeconds} - how long to wait for all reconciliations (default 120)
+ *
+ */
+@PerformanceTest(type = PerformanceTest.END_TO_END)
+class ReconciliationThroughputE2E {
+
+ private static final Logger log = LoggerFactory.getLogger(ReconciliationThroughputE2E.class);
+
+ private static final String RESOURCE_NAME_PREFIX = "perf-resource-";
+ private static final String INITIAL_VALUE = "initial";
+ private static final String UPDATED_VALUE = "updated";
+
+ private static final int RESOURCE_COUNT = Integer.getInteger("performance.resourceCount", 100);
+ private static final Duration TIMEOUT =
+ Duration.ofSeconds(Integer.getInteger("performance.timeoutSeconds", 120));
+
+ @RegisterExtension
+ LocallyRunOperatorExtension operator =
+ LocallyRunOperatorExtension.builder().withReconciler(new PerformanceReconciler()).build();
+
+ @Test
+ void reconcilesBatchOfResourcesOnCreateAndUpdate(PerformanceTestResults results) {
+ results.param("resourceCount", RESOURCE_COUNT);
+
+ log.info("Creating {} resources", RESOURCE_COUNT);
+ var start = System.nanoTime();
+ for (int i = 0; i < RESOURCE_COUNT; i++) {
+ operator.create(resource(i, INITIAL_VALUE));
+ }
+ var createdAt = System.nanoTime();
+ awaitAllObserved(INITIAL_VALUE);
+ report(results, "create", start, createdAt, System.nanoTime());
+
+ log.info("Updating {} resources", RESOURCE_COUNT);
+ start = System.nanoTime();
+ for (var resource : operator.resources(PerformanceCustomResource.class).list().getItems()) {
+ resource.getSpec().setValue(UPDATED_VALUE);
+ operator.update(resource);
+ }
+ var updatedAt = System.nanoTime();
+ awaitAllObserved(UPDATED_VALUE);
+ report(results, "update", start, updatedAt, System.nanoTime());
+
+ var executions =
+ operator.getReconcilerOfType(PerformanceReconciler.class).getNumberOfExecutions();
+ log.info(
+ "Total reconciliations: {} for {} resources ({} per resource)",
+ executions,
+ RESOURCE_COUNT,
+ String.format("%.2f", executions / (double) RESOURCE_COUNT));
+ results.record("reconciliationsPerResource", executions / (double) RESOURCE_COUNT, "count");
+
+ // one reconciliation for the create, one for the update; more means redundant work
+ assertThat(executions).isGreaterThanOrEqualTo(2 * RESOURCE_COUNT);
+ }
+
+ private void awaitAllObserved(String expectedValue) {
+ await()
+ .atMost(TIMEOUT)
+ .pollInterval(Duration.ofMillis(100))
+ .untilAsserted(() -> assertThat(numberOfObserved(expectedValue)).isEqualTo(RESOURCE_COUNT));
+ }
+
+ private long numberOfObserved(String expectedValue) {
+ return operator.resources(PerformanceCustomResource.class).list().getItems().stream()
+ .filter(r -> r.getStatus() != null)
+ .filter(r -> expectedValue.equals(r.getStatus().getObservedValue()))
+ .filter(r -> r.getMetadata().getGeneration().equals(r.getStatus().getObservedGeneration()))
+ .count();
+ }
+
+ private void report(
+ PerformanceTestResults results, String phase, long start, long submitted, long end) {
+ var submitMs = (submitted - start) / 1_000_000;
+ var totalMs = (end - start) / 1_000_000;
+ log.info(
+ "{}: submitted {} resources in {} ms, all reconciled after {} ms ({} reconciliations/sec)",
+ phase,
+ RESOURCE_COUNT,
+ submitMs,
+ totalMs,
+ String.format("%.1f", RESOURCE_COUNT * 1000.0 / Math.max(totalMs, 1)));
+
+ results
+ .recordElapsed(phase + ".submit", start, submitted)
+ .recordElapsed(phase, start, end)
+ .record(
+ phase + ".throughput",
+ RESOURCE_COUNT * 1_000_000_000.0 / (end - start),
+ "reconciliations/s");
+ }
+
+ private PerformanceCustomResource resource(int index, String value) {
+ var resource = new PerformanceCustomResource();
+ resource.setMetadata(new ObjectMetaBuilder().withName(RESOURCE_NAME_PREFIX + index).build());
+ var spec = new PerformanceCustomResourceSpec();
+ spec.setValue(value);
+ resource.setSpec(spec);
+ return resource;
+ }
+}
diff --git a/performance-tests/e2e/src/test/resources/log4j2-test.xml b/performance-tests/e2e/src/test/resources/log4j2-test.xml
new file mode 100644
index 0000000000..06faebd0ea
--- /dev/null
+++ b/performance-tests/e2e/src/test/resources/log4j2-test.xml
@@ -0,0 +1,32 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/performance-tests/jmh/pom.xml b/performance-tests/jmh/pom.xml
new file mode 100644
index 0000000000..e0164c901b
--- /dev/null
+++ b/performance-tests/jmh/pom.xml
@@ -0,0 +1,183 @@
+
+
+
+ 4.0.0
+
+ io.javaoperatorsdk
+ performance-tests
+ 999-SNAPSHOT
+ ../pom.xml
+
+
+ performance-tests-jmh
+ jar
+ Operator SDK - Performance Tests - JMH
+ JMH micro-benchmarks and in-process throughput tests for the Operator SDK
+
+
+
+
+ io.javaoperatorsdk
+ operator-framework-core
+
+
+ io.javaoperatorsdk
+ operator-framework-core
+ test-jar
+
+
+
+
+ io.javaoperatorsdk
+ performance-tests-reporting
+ ${project.version}
+
+
+
+
+ org.openjdk.jmh
+ jmh-core
+ ${jmh.version}
+
+
+ org.openjdk.jmh
+ jmh-generator-annprocess
+ ${jmh.version}
+ provided
+
+
+
+
+ org.mockito
+ mockito-core
+
+
+
+
+ io.fabric8
+ kubernetes-client
+
+
+ io.fabric8
+ kubernetes-httpclient-okhttp
+
+
+
+
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ test
+
+
+ org.assertj
+ assertj-core
+ test
+
+
+ org.awaitility
+ awaitility
+ test
+
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+ org.apache.logging.log4j
+ log4j-slf4j2-impl
+ test
+
+
+ org.apache.logging.log4j
+ log4j-core
+ test
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+
+
+
+ org.openjdk.jmh
+ jmh-generator-annprocess
+ ${jmh.version}
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ ${maven-shade-plugin.version}
+
+
+
+ shade
+
+ package
+
+ benchmarks
+ false
+
+
+
+ io.javaoperatorsdk.operator.performance.jmh.BenchmarkRunner
+
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+ ${skipPerformanceTests}
+
+ ${project.build.directory}/performance-results
+
+
+
+
+
+
diff --git a/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/performance/jmh/BenchmarkRunner.java b/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/performance/jmh/BenchmarkRunner.java
new file mode 100644
index 0000000000..c365bfcda5
--- /dev/null
+++ b/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/performance/jmh/BenchmarkRunner.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance.jmh;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+import org.openjdk.jmh.results.RunResult;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.CommandLineOptions;
+
+import io.javaoperatorsdk.operator.performance.results.Measurement;
+import io.javaoperatorsdk.operator.performance.results.PerformanceTest;
+import io.javaoperatorsdk.operator.performance.results.ResultsWriter;
+import io.javaoperatorsdk.operator.performance.results.TestResult;
+
+/**
+ * Main class of the benchmark jar. Takes the same arguments as {@link org.openjdk.jmh.Main} and in
+ * addition records the scores as performance test results, so that they can be stored and compared
+ * per commit.
+ *
+ * The scores are read from the objects the runner returns, not from the JMH result file, so
+ * whatever {@code -rf} / {@code -rff} produce stays exactly what JMH itself writes.
+ */
+public class BenchmarkRunner {
+
+ private static final String NO_PARAMS_MEASUREMENT = "score";
+
+ public static void main(String[] args) throws Exception {
+ var options = new CommandLineOptions(args);
+ if (options.shouldHelp()
+ || options.shouldList()
+ || options.shouldListWithParams()
+ || options.shouldListResultFormats()
+ || options.shouldListProfilers()) {
+ org.openjdk.jmh.Main.main(args);
+ return;
+ }
+ record(new Runner(options).run());
+ }
+
+ private static void record(Collection runResults) {
+ var writer = new ResultsWriter();
+ // a benchmark method has one result per parameter combination, all of them end up as
+ // measurements in the result file of that method
+ Map> measurementsByBenchmark = new LinkedHashMap<>();
+ for (RunResult runResult : runResults) {
+ var primary = runResult.getPrimaryResult();
+ var params = parameters(runResult);
+ measurementsByBenchmark
+ .computeIfAbsent(runResult.getParams().getBenchmark(), benchmark -> new ArrayList<>())
+ .add(
+ new Measurement(
+ measurementName(params), primary.getScore(), primary.getScoreUnit(), params));
+ }
+ measurementsByBenchmark.forEach(
+ (benchmark, measurements) -> {
+ var lastDot = benchmark.lastIndexOf('.');
+ writer.write(
+ TestResult.of(
+ PerformanceTest.JMH,
+ benchmark.substring(0, lastDot),
+ benchmark.substring(lastDot + 1),
+ writer.run(),
+ measurements));
+ });
+ }
+
+ private static Map parameters(RunResult runResult) {
+ var benchmarkParams = runResult.getParams();
+ Map params = new TreeMap<>();
+ for (Object key : benchmarkParams.getParamsKeys()) {
+ var name = String.valueOf(key);
+ params.put(name, benchmarkParams.getParam(name));
+ }
+ return params;
+ }
+
+ private static String measurementName(Map params) {
+ if (params.isEmpty()) {
+ return NO_PARAMS_MEASUREMENT;
+ }
+ return params.entrySet().stream()
+ .map(param -> param.getKey() + "=" + param.getValue())
+ .collect(Collectors.joining(","));
+ }
+}
diff --git a/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessorBenchmark.java b/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessorBenchmark.java
new file mode 100644
index 0000000000..9e63c55d1d
--- /dev/null
+++ b/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessorBenchmark.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.processing.event;
+
+import java.time.Duration;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+
+import io.javaoperatorsdk.operator.TestUtils;
+import io.javaoperatorsdk.operator.api.config.BaseConfigurationService;
+import io.javaoperatorsdk.operator.api.config.ControllerConfiguration;
+import io.javaoperatorsdk.operator.processing.event.rate.LinearRateLimiter;
+import io.javaoperatorsdk.operator.processing.event.source.ResourceAction;
+import io.javaoperatorsdk.operator.processing.event.source.controller.ControllerEventSource;
+import io.javaoperatorsdk.operator.processing.event.source.controller.ResourceEvent;
+import io.javaoperatorsdk.operator.processing.event.source.timer.TimerEventSource;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(2)
+@State(Scope.Thread)
+@SuppressWarnings({"rawtypes", "unchecked"})
+public class EventProcessorBenchmark {
+
+ private EventProcessor eventProcessor;
+
+ @Setup(Level.Iteration)
+ public void setup() {
+ ReconciliationDispatcher reconciliationDispatcher = mock(ReconciliationDispatcher.class);
+ when(reconciliationDispatcher.handleExecution(any()))
+ .thenReturn(PostExecutionControl.defaultDispatch());
+
+ EventSourceManager eventSourceManager = mock(EventSourceManager.class);
+ ControllerEventSource controllerEventSource = mock(ControllerEventSource.class);
+ when(eventSourceManager.getControllerEventSource()).thenReturn(controllerEventSource);
+
+ TimerEventSource retryTimerEventSource = mock(TimerEventSource.class);
+
+ ControllerConfiguration config = mock(ControllerConfiguration.class);
+ when(config.getName()).thenReturn("benchmark");
+ when(config.getRetry()).thenReturn(null);
+ when(config.getRateLimiter()).thenReturn(LinearRateLimiter.deactivatedRateLimiter());
+ when(config.maxReconciliationInterval()).thenReturn(Optional.of(Duration.ofHours(1)));
+ when(config.getConfigurationService()).thenReturn(new BaseConfigurationService());
+ when(config.triggerReconcilerOnAllEvents()).thenReturn(false);
+
+ eventProcessor =
+ spy(new EventProcessor(config, reconciliationDispatcher, eventSourceManager, null));
+ when(eventProcessor.retryEventSource()).thenReturn(retryTimerEventSource);
+ eventProcessor.start();
+ }
+
+ @TearDown(Level.Iteration)
+ public void tearDown() {
+ eventProcessor.stop();
+ }
+
+ @Benchmark
+ public void handleSingleEvent(Blackhole bh) {
+ var cr = TestUtils.testCustomResource();
+ ResourceID resourceID = ResourceID.fromResource(cr);
+ ResourceEvent event = new ResourceEvent(ResourceAction.UPDATED, resourceID, cr);
+ eventProcessor.handleEvent(event);
+ }
+}
diff --git a/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java b/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java
new file mode 100644
index 0000000000..72e2029aca
--- /dev/null
+++ b/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.processing.event.source.informer;
+
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+
+import io.fabric8.kubernetes.api.model.ConfigMap;
+import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
+
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(2)
+@State(Scope.Benchmark)
+public class PrimaryToSecondaryIndexBenchmark {
+
+ @Param({"10", "100", "1000"})
+ private int indexSize;
+
+ private DefaultPrimaryToSecondaryIndex index;
+ private ConfigMap sampleResource;
+ private ResourceID samplePrimaryId;
+
+ @Setup(Level.Iteration)
+ public void setup() {
+ ResourceID primaryId = new ResourceID("primary-0", "default");
+ index = new DefaultPrimaryToSecondaryIndex<>(resource -> Set.of(primaryId));
+
+ for (int i = 0; i < indexSize; i++) {
+ ConfigMap cm = createConfigMap("secondary-" + i);
+ index.onAddOrUpdate(cm, null);
+ }
+
+ sampleResource = createConfigMap("new-secondary");
+ samplePrimaryId = primaryId;
+ }
+
+ @Benchmark
+ public void onAddOrUpdate() {
+ index.onAddOrUpdate(sampleResource, null);
+ }
+
+ @Benchmark
+ public void getSecondaryResources(Blackhole bh) {
+ bh.consume(index.getSecondaryResources(samplePrimaryId));
+ }
+
+ @Benchmark
+ public void onDelete() {
+ index.onDelete(sampleResource);
+ }
+
+ @Benchmark
+ public void addThenDelete() {
+ index.onAddOrUpdate(sampleResource, null);
+ index.onDelete(sampleResource);
+ }
+
+ private static ConfigMap createConfigMap(String name) {
+ ConfigMap cm = new ConfigMap();
+ cm.setMetadata(
+ new ObjectMetaBuilder()
+ .withName(name)
+ .withNamespace("default")
+ .withResourceVersion("1")
+ .build());
+ return cm;
+ }
+}
diff --git a/performance-tests/jmh/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java b/performance-tests/jmh/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java
new file mode 100644
index 0000000000..89fd873f8f
--- /dev/null
+++ b/performance-tests/jmh/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.processing.event;
+
+import java.time.Duration;
+import java.util.Optional;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.javaoperatorsdk.operator.TestUtils;
+import io.javaoperatorsdk.operator.api.config.BaseConfigurationService;
+import io.javaoperatorsdk.operator.api.config.ControllerConfiguration;
+import io.javaoperatorsdk.operator.performance.results.PerformanceTest;
+import io.javaoperatorsdk.operator.performance.results.PerformanceTestResults;
+import io.javaoperatorsdk.operator.processing.event.rate.LinearRateLimiter;
+import io.javaoperatorsdk.operator.processing.event.source.ResourceAction;
+import io.javaoperatorsdk.operator.processing.event.source.controller.ControllerEventSource;
+import io.javaoperatorsdk.operator.processing.event.source.controller.ResourceEvent;
+import io.javaoperatorsdk.operator.processing.event.source.timer.TimerEventSource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+@PerformanceTest(type = PerformanceTest.IN_PROCESS)
+@SuppressWarnings({"rawtypes", "unchecked"})
+class EventProcessorThroughputTest {
+
+ private static final Logger log = LoggerFactory.getLogger(EventProcessorThroughputTest.class);
+ private static final int EVENT_COUNT = 1000;
+
+ private EventProcessor eventProcessor;
+
+ @BeforeEach
+ void setup() {
+ ReconciliationDispatcher reconciliationDispatcher = mock(ReconciliationDispatcher.class);
+ when(reconciliationDispatcher.handleExecution(any()))
+ .thenReturn(PostExecutionControl.defaultDispatch());
+
+ EventSourceManager eventSourceManager = mock(EventSourceManager.class);
+ ControllerEventSource controllerEventSource = mock(ControllerEventSource.class);
+ when(eventSourceManager.getControllerEventSource()).thenReturn(controllerEventSource);
+
+ TimerEventSource retryTimerEventSource = mock(TimerEventSource.class);
+
+ ControllerConfiguration config = mock(ControllerConfiguration.class);
+ when(config.getName()).thenReturn("throughput-test");
+ when(config.getRetry()).thenReturn(null);
+ when(config.getRateLimiter()).thenReturn(LinearRateLimiter.deactivatedRateLimiter());
+ when(config.maxReconciliationInterval()).thenReturn(Optional.of(Duration.ofHours(1)));
+ when(config.getConfigurationService()).thenReturn(new BaseConfigurationService());
+ when(config.triggerReconcilerOnAllEvents()).thenReturn(false);
+
+ eventProcessor =
+ spy(new EventProcessor(config, reconciliationDispatcher, eventSourceManager, null));
+ when(eventProcessor.retryEventSource()).thenReturn(retryTimerEventSource);
+ eventProcessor.start();
+ }
+
+ @AfterEach
+ void tearDown() {
+ eventProcessor.stop();
+ }
+
+ @Test
+ void shouldProcessEventsWithAcceptableThroughput(PerformanceTestResults results) {
+ long startTime = System.nanoTime();
+
+ for (int i = 0; i < EVENT_COUNT; i++) {
+ var cr = TestUtils.testCustomResource();
+ ResourceID resourceID = ResourceID.fromResource(cr);
+ ResourceEvent event = new ResourceEvent(ResourceAction.UPDATED, resourceID, cr);
+ eventProcessor.handleEvent(event);
+ }
+
+ long elapsedNanos = System.nanoTime() - startTime;
+ long elapsedMs = elapsedNanos / 1_000_000;
+ double eventsPerSecond = EVENT_COUNT * 1000.0 / elapsedMs;
+
+ log.info(
+ "EventProcessor throughput: {} events in {} ms ({} events/sec)",
+ EVENT_COUNT,
+ elapsedMs,
+ String.format("%.1f", eventsPerSecond));
+
+ results
+ .param("eventCount", EVENT_COUNT)
+ .recordMillis("handleEvent", elapsedNanos / 1_000_000.0)
+ .record("handleEvent.throughput", eventsPerSecond, "events/s");
+
+ assertThat(eventsPerSecond)
+ .as("Event processing throughput should exceed 100 events/sec")
+ .isGreaterThan(100.0);
+ }
+}
diff --git a/performance-tests/jmh/src/test/resources/log4j2-test.xml b/performance-tests/jmh/src/test/resources/log4j2-test.xml
new file mode 100644
index 0000000000..651b49c24a
--- /dev/null
+++ b/performance-tests/jmh/src/test/resources/log4j2-test.xml
@@ -0,0 +1,32 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/performance-tests/pom.xml b/performance-tests/pom.xml
new file mode 100644
index 0000000000..0916a48b3b
--- /dev/null
+++ b/performance-tests/pom.xml
@@ -0,0 +1,44 @@
+
+
+
+ 4.0.0
+
+ io.javaoperatorsdk
+ java-operator-sdk
+ 999-SNAPSHOT
+ ../pom.xml
+
+
+ performance-tests
+ pom
+ Operator SDK - Performance Tests
+ Aggregator for the performance test modules of the Operator SDK
+
+
+ reporting
+ jmh
+ e2e
+
+
+
+
+ true
+
+
diff --git a/performance-tests/reporting/pom.xml b/performance-tests/reporting/pom.xml
new file mode 100644
index 0000000000..469da84516
--- /dev/null
+++ b/performance-tests/reporting/pom.xml
@@ -0,0 +1,121 @@
+
+
+
+ 4.0.0
+
+ io.javaoperatorsdk
+ performance-tests
+ 999-SNAPSHOT
+ ../pom.xml
+
+
+ performance-tests-reporting
+ jar
+ Operator SDK - Performance Tests - Reporting
+ Records performance test results as JSON so they can be stored, compared and visualized
+
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ test
+
+
+ org.assertj
+ assertj-core
+ test
+
+
+
+
+
+
+
+ io.github.git-commit-id
+ git-commit-id-maven-plugin
+ ${git-commit-id-maven-plugin.version}
+
+ true
+ ${project.build.outputDirectory}/performance-tests-git.properties
+
+ ^git.commit.id.(abbrev|full)$
+ ^git.commit.time$
+ ^git.branch$
+
+ full
+ yyyy-MM-dd'T'HH:mm:ss'Z'
+ UTC
+
+ false
+
+
+
+ get-the-git-infos
+
+ revision
+
+ initialize
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ ${maven-shade-plugin.version}
+
+
+
+ shade
+
+ package
+
+ performance-results-tools
+ false
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+
+
+
+
+
+
+
+
+
+
diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Json.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Json.java
new file mode 100644
index 0000000000..d5e948e28c
--- /dev/null
+++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Json.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance.results;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
+/** Single place configuring how result files are read and written. */
+final class Json {
+
+ private static final ObjectMapper MAPPER =
+ new ObjectMapper()
+ .enable(SerializationFeature.INDENT_OUTPUT)
+ // keeps diffs on the results branch stable
+ .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
+
+ private Json() {}
+
+ static void write(Path file, Object value) throws IOException {
+ var directory = file.getParent();
+ if (directory != null) {
+ Files.createDirectories(directory);
+ }
+ Files.writeString(
+ file, MAPPER.writeValueAsString(value) + System.lineSeparator(), StandardCharsets.UTF_8);
+ }
+
+ static T read(Path file, Class type) throws IOException {
+ return MAPPER.readValue(Files.readString(file, StandardCharsets.UTF_8), type);
+ }
+}
diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Measurement.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Measurement.java
new file mode 100644
index 0000000000..dd2e8ccc35
--- /dev/null
+++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Measurement.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance.results;
+
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+/**
+ * A single measured value of a performance test.
+ *
+ * @param name identifies the measurement within a test, e.g. {@code create} or {@code total}
+ * @param value the measured value
+ * @param unit unit of the value, e.g. {@code ms} or {@code ops/s}
+ * @param params what the measurement was parameterized with, e.g. {@code resourceCount=100}
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public record Measurement(String name, double value, String unit, Map params) {
+
+ public static final String MILLISECONDS = "ms";
+
+ public Measurement(String name, double value, String unit) {
+ this(name, value, unit, Map.of());
+ }
+}
diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceResultsExtension.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceResultsExtension.java
new file mode 100644
index 0000000000..0a8495238a
--- /dev/null
+++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceResultsExtension.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance.results;
+
+import java.util.ArrayList;
+import java.util.Objects;
+
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolver;
+
+/**
+ * Records the measurements of tests annotated with {@link PerformanceTest}. Registered by that
+ * annotation, so it is not meant to be used with {@code @ExtendWith} directly.
+ *
+ * Results of failed tests are not recorded, a partial measurement would only distort the
+ * comparison with other runs.
+ */
+public class PerformanceResultsExtension
+ implements BeforeEachCallback, AfterEachCallback, ParameterResolver {
+
+ private static final ExtensionContext.Namespace NAMESPACE =
+ ExtensionContext.Namespace.create(PerformanceResultsExtension.class);
+ private static final String RESULTS = "results";
+ private static final String START_TIME = "startTime";
+ private static final String TOTAL_MEASUREMENT = "total";
+
+ private final ResultsWriter writer;
+
+ public PerformanceResultsExtension() {
+ this(new ResultsWriter());
+ }
+
+ PerformanceResultsExtension(ResultsWriter writer) {
+ this.writer = writer;
+ }
+
+ @Override
+ public void beforeEach(ExtensionContext context) {
+ var store = context.getStore(NAMESPACE);
+ store.put(RESULTS, new PerformanceTestResults());
+ store.put(START_TIME, System.nanoTime());
+ }
+
+ @Override
+ public void afterEach(ExtensionContext context) {
+ var endTime = System.nanoTime();
+ if (context.getExecutionException().isPresent()) {
+ return;
+ }
+ var store = context.getStore(NAMESPACE);
+ var results = Objects.requireNonNull(store.get(RESULTS, PerformanceTestResults.class));
+ var startTime = Objects.requireNonNull(store.get(START_TIME, Long.class));
+ var measurements = new ArrayList<>(results.measurements());
+ measurements.add(
+ new Measurement(
+ TOTAL_MEASUREMENT, (endTime - startTime) / 1_000_000.0, Measurement.MILLISECONDS));
+
+ writer.write(
+ TestResult.of(
+ typeOf(context),
+ context.getRequiredTestClass().getName(),
+ context.getRequiredTestMethod().getName(),
+ writer.run(),
+ measurements));
+ }
+
+ @Override
+ public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext context) {
+ return PerformanceTestResults.class.equals(parameterContext.getParameter().getType());
+ }
+
+ @Override
+ public Object resolveParameter(ParameterContext parameterContext, ExtensionContext context) {
+ return context.getStore(NAMESPACE).get(RESULTS, PerformanceTestResults.class);
+ }
+
+ private static String typeOf(ExtensionContext context) {
+ var testClass = context.getRequiredTestClass();
+ var annotation = testClass.getAnnotation(PerformanceTest.class);
+ if (annotation == null) {
+ throw new IllegalStateException(
+ testClass.getName() + " is not annotated with @" + PerformanceTest.class.getSimpleName());
+ }
+ return annotation.type();
+ }
+}
diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTest.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTest.java
new file mode 100644
index 0000000000..1040f3f455
--- /dev/null
+++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance.results;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * Marks a test class as a performance test, its measurements are recorded by {@link
+ * PerformanceResultsExtension}. Test methods can take a {@link PerformanceTestResults} parameter to
+ * add measurements, the duration of the test method itself is always recorded.
+ */
+@Documented
+@Inherited
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+@ExtendWith(PerformanceResultsExtension.class)
+public @interface PerformanceTest {
+
+ /** JMH benchmarks. */
+ String JMH = "jmh";
+
+ /** Tests measuring the SDK in process, without a cluster. */
+ String IN_PROCESS = "in-process";
+
+ /** Tests measuring an operator running against a real cluster. */
+ String END_TO_END = "e2e";
+
+ /**
+ * Category of the performance test, becomes the directory the results are stored in. Use one of
+ * the constants of this annotation unless a new category is added.
+ */
+ String type();
+}
diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTestResults.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTestResults.java
new file mode 100644
index 0000000000..d5aca1c5ed
--- /dev/null
+++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTestResults.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance.results;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Collects the measurements of a single test method. Injected as a test method parameter by {@link
+ * PerformanceResultsExtension}.
+ */
+public class PerformanceTestResults {
+
+ private final List measurements = new ArrayList<>();
+ private final Map params = new LinkedHashMap<>();
+
+ /**
+ * Adds a parameter describing the scenario, for example the number of resources. It is attached
+ * to every measurement recorded afterwards, so that results are only compared with results of the
+ * same scenario.
+ */
+ public PerformanceTestResults param(String name, Object value) {
+ params.put(name, String.valueOf(value));
+ return this;
+ }
+
+ public PerformanceTestResults record(String name, double value, String unit) {
+ measurements.add(new Measurement(name, value, unit, Map.copyOf(params)));
+ return this;
+ }
+
+ public PerformanceTestResults recordDuration(String name, Duration duration) {
+ return recordMillis(name, duration.toNanos() / 1_000_000.0);
+ }
+
+ /** Records the duration between two {@link System#nanoTime()} readings. */
+ public PerformanceTestResults recordElapsed(String name, long startNanos, long endNanos) {
+ return recordMillis(name, (endNanos - startNanos) / 1_000_000.0);
+ }
+
+ public PerformanceTestResults recordMillis(String name, double millis) {
+ return record(name, millis, Measurement.MILLISECONDS);
+ }
+
+ public List measurements() {
+ return List.copyOf(measurements);
+ }
+}
diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndex.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndex.java
new file mode 100644
index 0000000000..b94031f990
--- /dev/null
+++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndex.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance.results;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+/**
+ * Content of {@value ResultsIndexer#INDEX_FILE}: all runs stored on the results branch, ordered
+ * from oldest to newest commit. Lets tooling read the results in order without looking at the git
+ * history of the branch they are stored on.
+ *
+ * Holds nothing that changes between two regenerations of the same runs, so that re-indexing
+ * without new results does not produce a commit.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public record ResultsIndex(List runs) {
+
+ /**
+ * @param directory name of the directory the results of the run are in
+ * @param types the categories of performance tests that produced results in this run
+ */
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public record Run(
+ String directory,
+ String commit,
+ String commitAbbrev,
+ String commitTimestamp,
+ String branch,
+ List types) {}
+}
diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndexer.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndexer.java
new file mode 100644
index 0000000000..5f1418f932
--- /dev/null
+++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndexer.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance.results;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Stream;
+
+/**
+ * Regenerates {@value #INDEX_FILE} for a results directory by scanning the run directories in it.
+ * Rebuilding instead of appending keeps the index correct no matter in which order runs are added,
+ * and avoids conflicts when several jobs contribute results for the same commit.
+ *
+ * Invoked by CI after the results of a commit have been merged onto the results branch:
+ *
+ *
+ * java -cp performance-results-tools.jar \
+ * io.javaoperatorsdk.operator.performance.results.ResultsIndexer performance-tests/results
+ *
+ */
+public final class ResultsIndexer {
+
+ public static final String INDEX_FILE = "index.json";
+
+ private ResultsIndexer() {}
+
+ public static void main(String[] args) throws IOException {
+ if (args.length != 1) {
+ System.err.println("Usage: " + ResultsIndexer.class.getName() + " ");
+ System.exit(1);
+ return;
+ }
+ var resultsDirectory = Path.of(args[0]);
+ var index = index(resultsDirectory);
+ var indexFile = resultsDirectory.resolve(INDEX_FILE);
+ Json.write(indexFile, index);
+ System.out.println("Indexed " + index.runs().size() + " run(s) into " + indexFile);
+ }
+
+ /** Reads all runs of a results directory, ordered from the oldest to the newest commit. */
+ public static ResultsIndex index(Path resultsDirectory) throws IOException {
+ var runs = new ArrayList();
+ try (var directories = Files.list(resultsDirectory)) {
+ for (var directory : directories.filter(Files::isDirectory).toList()) {
+ var run = read(directory);
+ if (run != null) {
+ runs.add(run);
+ }
+ }
+ }
+ runs.sort(
+ Comparator.comparing(ResultsIndex.Run::commitTimestamp)
+ .thenComparing(ResultsIndex.Run::directory));
+ return new ResultsIndex(runs);
+ }
+
+ private static ResultsIndex.Run read(Path directory) throws IOException {
+ var runFile = directory.resolve(ResultsWriter.RUN_FILE);
+ if (!Files.isRegularFile(runFile)) {
+ return null;
+ }
+ var metadata = Json.read(runFile, RunMetadata.class);
+ return new ResultsIndex.Run(
+ String.valueOf(directory.getFileName()),
+ metadata.commit(),
+ metadata.commitAbbrev(),
+ metadata.commitTimestamp(),
+ metadata.branch(),
+ types(directory));
+ }
+
+ private static List types(Path directory) throws IOException {
+ try (Stream content = Files.list(directory)) {
+ return content
+ .filter(Files::isDirectory)
+ .map(path -> String.valueOf(path.getFileName()))
+ .sorted()
+ .toList();
+ }
+ }
+}
diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsWriter.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsWriter.java
new file mode 100644
index 0000000000..8368e39a1e
--- /dev/null
+++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsWriter.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance.results;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Writes performance test results into the directory of the current run:
+ *
+ *
+ * <results dir>/<commit timestamp>-<short commit>/run.json
+ * <results dir>/<commit timestamp>-<short commit>/<type>/<test class>.<test method>.json
+ *
+ *
+ * The results directory defaults to {@code target/performance-results} and can be set with the
+ * {@value RunMetadata#RESULTS_DIR_PROPERTY} system property. CI collects those directories from all
+ * performance test jobs and merges them onto the results branch, which is why the layout below the
+ * results directory is the same as the one on that branch.
+ */
+public class ResultsWriter {
+
+ public static final String RUN_FILE = "run.json";
+ private static final String DEFAULT_RESULTS_DIR = "target/performance-results";
+
+ private static final Logger log = LoggerFactory.getLogger(ResultsWriter.class);
+
+ private final RunMetadata run;
+ private final Path runDirectory;
+
+ public ResultsWriter() {
+ this(defaultResultsDirectory(), RunMetadata.current());
+ }
+
+ public ResultsWriter(Path resultsDirectory, RunMetadata run) {
+ this.run = run;
+ this.runDirectory = resultsDirectory.resolve(run.directoryName());
+ }
+
+ public static Path defaultResultsDirectory() {
+ return Path.of(System.getProperty(RunMetadata.RESULTS_DIR_PROPERTY, DEFAULT_RESULTS_DIR));
+ }
+
+ public RunMetadata run() {
+ return run;
+ }
+
+ public Path runDirectory() {
+ return runDirectory;
+ }
+
+ /** Writes a result file, creating {@value #RUN_FILE} for the run if it is not there yet. */
+ public Path write(TestResult result) {
+ try {
+ var runFile = runDirectory.resolve(RUN_FILE);
+ if (!Files.exists(runFile)) {
+ Json.write(runFile, run);
+ }
+ var resultFile = runDirectory.resolve(result.fileName());
+ Json.write(resultFile, result);
+ log.info("Recorded {} result of {} to {}", result.type(), result.testMethod(), resultFile);
+ return resultFile;
+ } catch (IOException e) {
+ throw new UncheckedIOException("Cannot write results to " + runDirectory, e);
+ }
+ }
+}
diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/RunMetadata.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/RunMetadata.java
new file mode 100644
index 0000000000..adb67a9ddc
--- /dev/null
+++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/RunMetadata.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance.results;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Properties;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+/**
+ * Identifies the commit a set of results belongs to. Written as {@code run.json} next to the
+ * results of a run.
+ *
+ *
The values are taken from {@code performance-tests-git.properties}, generated at build time by
+ * the git-commit-id plugin, so they are also available when the results are recorded outside of
+ * Maven, for example from the shaded JMH benchmark jar. Every value can be overridden with a system
+ * property, which is what CI does for the branch, since a GitHub Actions checkout has a detached
+ * HEAD.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public record RunMetadata(
+ String commit,
+ String commitAbbrev,
+ String commitTimestamp,
+ String branch,
+ String runTimestamp) {
+
+ public static final String RESULTS_DIR_PROPERTY = "performance.results.dir";
+ public static final String COMMIT_PROPERTY = "performance.results.commit";
+ public static final String COMMIT_TIMESTAMP_PROPERTY = "performance.results.commitTimestamp";
+ public static final String BRANCH_PROPERTY = "performance.results.branch";
+
+ private static final Logger log = LoggerFactory.getLogger(RunMetadata.class);
+
+ private static final String PROPERTIES_RESOURCE = "/performance-tests-git.properties";
+ private static final String UNKNOWN = "unknown";
+ private static final int ABBREV_LENGTH = 7;
+
+ /**
+ * Name of the directory the results of this run are stored in. Prefixed with the commit timestamp
+ * so that runs sort chronologically without consulting the git history.
+ */
+ @JsonIgnore
+ public String directoryName() {
+ return compactTimestamp(commitTimestamp) + "-" + commitAbbrev;
+ }
+
+ public static RunMetadata current() {
+ var gitProperties = loadGitProperties();
+
+ var commit = value(COMMIT_PROPERTY, gitProperties.getProperty("git.commit.id.full"), UNKNOWN);
+ var runTimestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS).toString();
+
+ return new RunMetadata(
+ commit,
+ abbreviate(gitProperties.getProperty("git.commit.id.abbrev"), commit),
+ value(
+ COMMIT_TIMESTAMP_PROPERTY, gitProperties.getProperty("git.commit.time"), runTimestamp),
+ value(BRANCH_PROPERTY, gitProperties.getProperty("git.branch"), UNKNOWN),
+ runTimestamp);
+ }
+
+ private static Properties loadGitProperties() {
+ var properties = new Properties();
+ try (InputStream is = RunMetadata.class.getResourceAsStream(PROPERTIES_RESOURCE)) {
+ if (is == null) {
+ log.warn(
+ "{} not found on the classpath, results will not be associated with a commit",
+ PROPERTIES_RESOURCE);
+ } else {
+ properties.load(is);
+ }
+ } catch (IOException e) {
+ log.warn("Cannot read {}", PROPERTIES_RESOURCE, e);
+ }
+ return properties;
+ }
+
+ private static String value(String systemProperty, String fromGit, String fallback) {
+ var override = System.getProperty(systemProperty);
+ if (isSet(override)) {
+ return override;
+ }
+ return isSet(fromGit) ? fromGit : fallback;
+ }
+
+ private static String abbreviate(String fromGit, String commit) {
+ if (isSet(fromGit) && !isSet(System.getProperty(COMMIT_PROPERTY))) {
+ return fromGit;
+ }
+ return commit.length() > ABBREV_LENGTH ? commit.substring(0, ABBREV_LENGTH) : commit;
+ }
+
+ private static boolean isSet(String value) {
+ // the git plugin leaves unresolved placeholders in place when not run in a git checkout
+ return value != null && !value.isBlank() && !value.startsWith("$");
+ }
+
+ private static String compactTimestamp(String timestamp) {
+ return timestamp.replace("-", "").replace(":", "");
+ }
+}
diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/TestResult.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/TestResult.java
new file mode 100644
index 0000000000..f8f1d621e4
--- /dev/null
+++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/TestResult.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance.results;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+/**
+ * All measurements of a single performance test, the content of one result file.
+ *
+ * @param type category of the performance test, see {@link PerformanceTest#type()}
+ * @param testClass fully qualified name of the test, for JMH the benchmark class
+ * @param testMethod test or benchmark method the measurements belong to
+ * @param commitAbbrev repeated from {@code run.json} so a single file is self describing
+ * @param commitTimestamp repeated from {@code run.json} so a single file is self describing
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public record TestResult(
+ String type,
+ String testClass,
+ String testMethod,
+ String commitAbbrev,
+ String commitTimestamp,
+ List measurements) {
+
+ public static TestResult of(
+ String type,
+ String testClass,
+ String testMethod,
+ RunMetadata run,
+ List measurements) {
+ return new TestResult(
+ type, testClass, testMethod, run.commitAbbrev(), run.commitTimestamp(), measurements);
+ }
+
+ /** Name of the result file, relative to the directory of the run. */
+ @JsonIgnore
+ public String fileName() {
+ return type + "/" + testClass + "." + testMethod + ".json";
+ }
+}
diff --git a/performance-tests/reporting/src/test/java/io/javaoperatorsdk/operator/performance/results/ResultsWriterTest.java b/performance-tests/reporting/src/test/java/io/javaoperatorsdk/operator/performance/results/ResultsWriterTest.java
new file mode 100644
index 0000000000..08f63cdf24
--- /dev/null
+++ b/performance-tests/reporting/src/test/java/io/javaoperatorsdk/operator/performance/results/ResultsWriterTest.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.performance.results;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class ResultsWriterTest {
+
+ private static final RunMetadata RUN =
+ new RunMetadata(
+ "abc123def456", "abc123d", "2026-08-25T12:00:00Z", "main", "2026-08-25T13:00:00Z");
+
+ @Test
+ void storesResultsUnderADirectoryNamedAfterTheCommit(@TempDir Path resultsDirectory) {
+ var writer = new ResultsWriter(resultsDirectory, RUN);
+
+ writer.write(
+ TestResult.of(
+ PerformanceTest.END_TO_END,
+ "com.acme.ThroughputE2E",
+ "measuresThroughput",
+ RUN,
+ List.of(new Measurement("create", 12.5, "ms", Map.of("resourceCount", "100")))));
+
+ var runDirectory = resultsDirectory.resolve("20260825T120000Z-abc123d");
+ assertThat(runDirectory.resolve(ResultsWriter.RUN_FILE)).exists();
+ assertThat(runDirectory.resolve("e2e/com.acme.ThroughputE2E.measuresThroughput.json")).exists();
+ }
+
+ @Test
+ void indexListsRunsOldestFirstWithTheirTypes(@TempDir Path resultsDirectory) throws IOException {
+ var older =
+ new RunMetadata("1", "aaaaaaa", "2026-08-24T10:00:00Z", "main", "2026-08-26T10:00:00Z");
+ var newer =
+ new RunMetadata("2", "bbbbbbb", "2026-08-25T10:00:00Z", "next", "2026-08-25T10:00:00Z");
+ // written newest first on purpose, the index has to order by commit, not by insertion
+ write(resultsDirectory, newer, PerformanceTest.JMH);
+ write(resultsDirectory, older, PerformanceTest.END_TO_END);
+ write(resultsDirectory, older, PerformanceTest.IN_PROCESS);
+
+ var index = ResultsIndexer.index(resultsDirectory);
+
+ assertThat(index.runs())
+ .map(ResultsIndex.Run::commitAbbrev)
+ .containsExactly("aaaaaaa", "bbbbbbb");
+ assertThat(index.runs().get(0).types()).containsExactly("e2e", "in-process");
+ assertThat(index.runs().get(0).branch()).isEqualTo("main");
+ assertThat(index.runs().get(1).types()).containsExactly("jmh");
+ }
+
+ @Test
+ void indexIgnoresDirectoriesWithoutRunMetadata(@TempDir Path resultsDirectory)
+ throws IOException {
+ write(resultsDirectory, RUN, PerformanceTest.JMH);
+ java.nio.file.Files.createDirectory(resultsDirectory.resolve("not-a-run"));
+
+ assertThat(ResultsIndexer.index(resultsDirectory).runs()).hasSize(1);
+ }
+
+ private static void write(Path resultsDirectory, RunMetadata run, String type) {
+ new ResultsWriter(resultsDirectory, run)
+ .write(
+ TestResult.of(
+ type,
+ "com.acme.SomeTest",
+ "someMeasurement",
+ run,
+ List.of(new Measurement("total", 1.0, "ms"))));
+ }
+}
diff --git a/pom.xml b/pom.xml
index e9b130430e..70ed83c3d1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -55,6 +55,7 @@
bootstrapper-maven-plugin
test-index-processor
migration
+ performance-tests
@@ -86,6 +87,8 @@
0.9.14
2.22.0
4.17
+
+ 2.21.4
2.11
3.15.0
@@ -104,6 +107,10 @@
3.5.2
3.9.0
4.10.3.0
+ 3.6.0
+ 1.37
+
+ true
@@ -178,6 +185,11 @@
java-diff-utils
${java.diff.version}
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ ${jackson.version}
+
org.slf4j
slf4j-api
@@ -209,6 +221,12 @@
operator-framework-core
${project.version}
+
+ io.javaoperatorsdk
+ operator-framework-core
+ ${project.version}
+ test-jar
+
io.javaoperatorsdk
operator-framework
@@ -647,5 +665,11 @@
+
+ performance-tests
+
+ false
+
+
diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml
index f6107ac4ea..fd8f6e22c4 100644
--- a/spotbugs-exclude.xml
+++ b/spotbugs-exclude.xml
@@ -17,6 +17,11 @@
-->
+
+
+
+
+