[WIP] Performance tests initial structure

.github/workflows/performance-tests.yml

@@ -0,0 +1,174 @@
+name: Performance Tests
+
+env:
+  MAVEN_ARGS: -V -ntp -e
+  # branch the results of every run are stored on, created on the first run
+  RESULTS_BRANCH: performance-test-results
+  RESULTS_PATH: performance-tests/results
+
+on:
+  push:
+    branches:
+      - main
+      - next
+    paths:
+      - 'operator-framework-core/src/main/**'
+      - 'performance-tests/**'
+  workflow_dispatch:
+
+jobs:
+  jmh-benchmarks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up Java and Maven
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: 17
+          cache: 'maven'
+      - name: Build project
+        run: ./mvnw ${MAVEN_ARGS} install -DskipTests --file pom.xml
+      - name: Run JMH benchmarks
+        run: |
+          java -Dperformance.results.dir=performance-tests/jmh/target/performance-results \
+            -Dperformance.results.branch=${{ github.ref_name }} \
+            -jar performance-tests/jmh/target/benchmarks.jar \
+            -rf json -rff performance-tests/jmh/target/jmh-result.json
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: jmh-result
+          path: performance-tests/jmh/target/jmh-result.json
+      - name: Upload performance results
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance-results-jmh
+          path: performance-tests/jmh/target/performance-results
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        if: github.ref == 'refs/heads/main'
+        with:
+          tool: 'jmh'
+          output-file-path: performance-tests/jmh/target/jmh-result.json
+          auto-push: true
+          alert-threshold: '130%'
+          comment-on-alert: true
+          fail-on-alert: false
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: dev/bench
+
+  in-process-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up Java and Maven
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: 17
+          cache: 'maven'
+      - name: Run in-process performance tests
+        run: |
+          ./mvnw ${MAVEN_ARGS} install -DskipTests --file pom.xml
+          ./mvnw ${MAVEN_ARGS} verify -P performance-tests \
+            -Dperformance.results.branch=${{ github.ref_name }} \
+            -pl performance-tests/jmh --file pom.xml
+      - name: Upload performance results
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance-results-in-process
+          path: performance-tests/jmh/target/performance-results
+
+  e2e-performance-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up Java and Maven
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: 17
+          cache: 'maven'
+      - name: Set up Minikube
+        uses: manusa/actions-setup-minikube@v2.18.0
+        with:
+          minikube version: 'v1.38.1'
+          kubernetes version: 'v1.35.2'
+          github token: ${{ github.token }}
+      - name: Run performance tests against the cluster
+        run: |
+          ./mvnw ${MAVEN_ARGS} install -DskipTests --file pom.xml
+          ./mvnw ${MAVEN_ARGS} verify -P performance-tests \
+            -Dperformance.results.branch=${{ github.ref_name }} \
+            -pl performance-tests/e2e --file pom.xml
+      - name: Upload performance results
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance-results-e2e
+          path: performance-tests/e2e/target/performance-results
+
+  publish-results:
+    needs: [jmh-benchmarks, in-process-tests, e2e-performance-tests]
+    # results of a partially failed run would be misleading when comparing commits, and only the
+    # long living branches are worth keeping a history for
+    if: github.ref_name == 'main' || github.ref_name == 'next'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up Java and Maven
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: 17
+          cache: 'maven'
+      - name: Build the results tooling
+        run: |
+          ./mvnw ${MAVEN_ARGS} package -Dmaven.test.skip=true \
+            -pl performance-tests/reporting -am --file pom.xml
+      - name: Download the results of all performance test jobs
+        uses: actions/download-artifact@v4
+        with:
+          pattern: performance-results-*
+          # all jobs of a run write into the same directory, named after the commit
+          merge-multiple: true
+          path: collected-results
+      - name: Check out the results branch
+        env:
+          REPOSITORY_URL: https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git
+        run: |
+          set -euo pipefail
+          if git ls-remote --exit-code --heads "$REPOSITORY_URL" "$RESULTS_BRANCH" > /dev/null; then
+            git clone --depth 1 --branch "$RESULTS_BRANCH" "$REPOSITORY_URL" results-branch
+          else
+            # first run: the branch holds only results, so it is started without any history
+            echo "$RESULTS_BRANCH does not exist yet, creating it"
+            git init results-branch
+            git -C results-branch remote add origin "$REPOSITORY_URL"
+            git -C results-branch checkout -b "$RESULTS_BRANCH"
+          fi
+      - name: Add the results of this commit
+        run: |
+          mkdir -p "results-branch/$RESULTS_PATH"
+          cp -R collected-results/. "results-branch/$RESULTS_PATH/"
+          java -cp performance-tests/reporting/target/performance-results-tools.jar \
+            io.javaoperatorsdk.operator.performance.results.ResultsIndexer \
+            "results-branch/$RESULTS_PATH"
+      - name: Push the results
+        working-directory: results-branch
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add "$RESULTS_PATH"
+          if git diff --cached --quiet; then
+            echo "No new results to store"
+            exit 0
+          fi
+          git commit -m "perf: results of ${{ github.ref_name }} at ${GITHUB_SHA:0:7}"
+          git push origin "HEAD:$RESULTS_BRANCH"

operator-framework-core/pom.xml

@@ -121,6 +121,17 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <!-- Used to generate the version / commit information -->
         <groupId>io.github.git-commit-id</groupId>

performance-tests/README.md

@@ -0,0 +1,100 @@
+# Performance Tests
+
+Performance related tests of the SDK. All of them are **skipped by default**; activate them with the
+`performance-tests` profile.
+
+| module      | what it measures                                              | needs a cluster |
+|-------------|---------------------------------------------------------------|-----------------|
+| `jmh`       | JMH micro-benchmarks and in-process throughput tests          | no              |
+| `e2e`       | end-to-end reconciliation of a real operator on a real cluster | yes             |
+| `reporting` | records the results of the above so they can be compared      | -               |
+
+## `jmh`
+
+JMH micro-benchmarks (`src/main/java`) and in-process throughput tests (`src/test/java`) for hot
+paths of the core.
+
+Run the throughput tests:
+
+```shell
+./mvnw verify -P performance-tests -pl performance-tests/jmh -am
+```
+
+Run the benchmarks (the shaded jar is built by a plain `install`):
+
+```shell
+./mvnw install -DskipTests
+java -jar performance-tests/jmh/target/benchmarks.jar
+```
+
+The jar takes the same arguments as JMH itself, it only wraps `org.openjdk.jmh.Main` to also record
+the scores as results, see `BenchmarkRunner`.
+
+## `e2e`
+
+Tests running a real operator against a real Kubernetes cluster, measuring end-to-end reconciliation
+throughput. They use the current kube context, so a cluster (minikube, kind, ...) has to be
+available.
+
+```shell
+./mvnw verify -P performance-tests -pl performance-tests/e2e -am
+```
+
+Scenario size can be tuned per test, see the javadoc of the individual tests, e.g.:
+
+```shell
+./mvnw verify -P performance-tests -pl performance-tests/e2e -am \
+  -Dperformance.resourceCount=1000 -Dperformance.timeoutSeconds=600
+```
+
+## `reporting`
+
+Records the measurements of the other two modules as JSON, so that the numbers of different commits
+can be compared and visualized. A test opts in by being annotated with `@PerformanceTest`, and
+receives a `PerformanceTestResults` parameter to add measurements to. The duration of the test method
+itself is always recorded, results of failed tests are not recorded at all.
+
+```java
+@PerformanceTest(type = PerformanceTest.END_TO_END)
+class SomethingE2E {
+
+  @Test
+  void measuresSomething(PerformanceTestResults results) {
+    results.param("resourceCount", 100).recordElapsed("create", start, end);
+  }
+}
+```
+
+Results are written below `target/performance-results` of the module, override with
+`-Dperformance.results.dir=...`.
+
+### How results are stored
+
+The `Performance Tests` workflow runs on every push to `main` and `next`, collects the results of all
+jobs and commits them to the **`performance-test-results`** branch, which holds nothing but results:
+
+```
+performance-tests/results/
+├── index.json                                    all runs, ordered oldest to newest commit
+└── 20260825T070249Z-b2bace4/                     <commit timestamp>-<short commit>
+    ├── run.json                                  commit, branch and timestamps of the run
+    ├── jmh/
+    │   └── <benchmark class>.<method>.json
+    ├── in-process/
+    │   └── <test class>.<test method>.json
+    └── e2e/
+        └── <test class>.<test method>.json
+```
+
+A run directory is named after the commit, prefixed with the **commit** timestamp (not the time the
+tests ran) so that runs sort chronologically without consulting the git history, and so that a re-run
+of a commit updates its results in place. `index.json` additionally lists the runs in order with
+their commit, branch and the categories of tests that produced results, which is what tooling should
+read instead of listing directories. It is regenerated from the run directories after every push:
+
+```shell
+java -cp performance-tests/reporting/target/performance-results-tools.jar \
+  io.javaoperatorsdk.operator.performance.results.ResultsIndexer performance-tests/results
+```
+
+The branch is created by the workflow on its first run and needs no manual setup.