ci: de-duplicate invariant gates across the test matrix

.github/workflows/package.yml

@@ -84,12 +84,6 @@ jobs:
             exit 1
           fi
 
-      - name: Install validation dependencies
-        run: python -m pip install ".[dev,typer,quality]"
-
-      - name: Run authoritative validation gate
-        run: ./tests/full_validate.sh
-
       - name: Validate changelog
         run: python scripts/validate_changelog.py
 

.github/workflows/tests.yml

@@ -2,6 +2,10 @@ name: Tests
 
 on:
   push:
+    branches:
+      - main
+    tags:
+      - "v*"
   pull_request:
 
 permissions:
@@ -35,10 +39,10 @@ jobs:
         uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install full validation dependencies
-        run: python -m pip install ".[dev,typer,quality]"
-      - name: Run authoritative validation gate
-        run: ./tests/full_validate.sh
+      - name: Install runtime test dependencies
+        run: python -m pip install ".[dev,typer]"
+      - name: Run platform and Python runtime suite
+        run: ./tests/full_validate.sh --gate runtime
 
   quality:
     name: Quality and security gates
@@ -55,37 +59,54 @@ jobs:
           python-version: "3.13"
       - name: Install quality dependencies
         run: python -m pip install ".[dev,typer,quality]"
-      - name: Run formatting and lint checks
-        run: |
-          ruff format --check lib/python/base_cli scripts examples tests
-          ruff check lib/python/base_cli scripts examples tests
-      - name: Run strict typing and documentation checks
-        run: |
-          python -m mypy --strict examples/typed_consumer.py
-          python -m mypy --strict lib/python/base_cli
-          python scripts/validate_docs.py
-          python scripts/validate_changelog.py
-          python scripts/validate_schemas.py
-          python scripts/validate_contract_fixtures.py
-          node scripts/validate_contract_fixtures.mjs
-          python scripts/generate_compatibility_dashboard.py --check
-          python scripts/benchmark_runtime.py --check
-          python -m compileall -q examples
-      - name: Run tests with coverage threshold
-        run: python -m pytest --cov=base_cli --cov-report=term-missing --cov-report=json:coverage.json --cov-fail-under=80
-      - name: Enforce high-risk module coverage floors
-        run: python scripts/validate_coverage.py coverage.json
-      - name: Run static security checks
-        run: |
-          bandit -q -r lib/python/base_cli scripts -lll -iii
-          # The project itself is installed from this checkout and may not be
-          # published to PyPI yet (for example, while validating a release PR).
-          # Audit every installed third-party package without asking pip-audit
-          # to resolve the unpublished project distribution.
-          python -m pip freeze \
-            | grep -Eiv '^base-cli([[:space:]]|$)' \
-            > "$RUNNER_TEMP/base-cli-audit-requirements.txt"
-          pip-audit --strict -r "$RUNNER_TEMP/base-cli-audit-requirements.txt"
+      - name: Validate repository baseline
+        run: ./tests/full_validate.sh --gate baseline
+      - name: Run tests and enforce coverage budgets
+        run: ./tests/full_validate.sh --gate coverage
+      - name: Enforce strict typing
+        run: ./tests/full_validate.sh --gate typing
+      - name: Enforce formatting and lint
+        run: ./tests/full_validate.sh --gate style
+      - name: Validate docs, schemas, contracts, and generated files
+        run: ./tests/full_validate.sh --gate contracts
+      - name: Run dependency and static security checks
+        run: ./tests/full_validate.sh --gate security
+
+  benchmarks:
+    name: Benchmark (${{ matrix.profile }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - profile: unix
+            os: ubuntu-latest
+          - profile: macos
+            os: macos-latest
+          - profile: windows
+            os: windows-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.13"
+      - name: Install declared benchmark comparators
+        run: python -m pip install ".[typer,benchmark]"
+      - name: Run calibrated comparative benchmark
+        env:
+          BASE_CLI_BENCHMARK_PLATFORM: ${{ matrix.profile }}
+          SOURCE_REVISION: ${{ github.sha }}
+        run: python scripts/benchmark_runtime.py --check --iterations 31 --output benchmark-results.json
+      - name: Retain dated machine-readable benchmark evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: base-cli-benchmark-${{ matrix.profile }}-${{ github.run_id }}
+          path: benchmark-results.json
+          if-no-files-found: ignore
+          retention-days: 90
 
   linux-distributions:
     name: Validate (${{ matrix.name }})
@@ -127,8 +148,7 @@ jobs:
               python3 -m venv /tmp/base-cli-venv
               /tmp/base-cli-venv/bin/python -m pip install ".[dev,typer]"
               . /tmp/base-cli-venv/bin/activate
-              python -m pip install ".[quality]"
-              ./tests/full_validate.sh
+              ./tests/full_validate.sh --gate runtime
             '
 
   wsl:
@@ -140,7 +160,7 @@ jobs:
         with:
           fetch-depth: 0
           fetch-tags: true
-      - name: Validate repository baseline inside WSL
+      - name: Run platform and Python runtime suite inside WSL
         shell: pwsh
         run: |
           $distros = (wsl --list --quiet 2>$null | Out-String)
@@ -150,4 +170,13 @@ jobs:
           $drive = $env:GITHUB_WORKSPACE.Substring(0, 1).ToLowerInvariant()
           $path = $env:GITHUB_WORKSPACE.Substring(2).Replace('\', '/')
           $linuxWorkspace = "/mnt/$drive$path"
-          wsl --distribution Ubuntu --user root -- bash -lc "set -eu; cd '$linuxWorkspace'; sed -i 's/\r$//' tests/full_validate.sh tests/validate.sh; apt-get update -qq; apt-get install -y -qq nodejs npm python3-venv python3.14-venv; python3 -m venv /tmp/base-cli-venv; . /tmp/base-cli-venv/bin/activate; python -m pip install '.[dev,typer,quality]'; export BASE_CLI_BENCHMARK_PLATFORM=wsl; bash tests/full_validate.sh"
+          $revision = $env:GITHUB_SHA
+          wsl --distribution Ubuntu --user root -- bash -lc "set -eu; cd '$linuxWorkspace'; sed -i 's/\r$//' tests/full_validate.sh tests/validate.sh; apt-get update -qq; apt-get install -y -qq nodejs npm python3-venv python3.14-venv; python3 -m venv /tmp/base-cli-venv; . /tmp/base-cli-venv/bin/activate; python -m pip install '.[dev,typer,benchmark]'; bash tests/full_validate.sh --gate runtime; BASE_CLI_BENCHMARK_PLATFORM=wsl SOURCE_REVISION='$revision' python scripts/benchmark_runtime.py --check --iterations 31 --output base-cli-benchmark-wsl.json"
+      - name: Retain WSL benchmark evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: base-cli-benchmark-wsl-${{ github.run_id }}
+          path: base-cli-benchmark-wsl.json
+          if-no-files-found: ignore
+          retention-days: 90

CHANGELOG.md

@@ -11,6 +11,11 @@ and versions are tracked in the repo-root `VERSION` file.
 
 - Continue compatibility hardening and adoption work for the next release.
 
+### Added
+
+- Publish versioned, comparative CLI benchmark reports with lifecycle and
+  feature scenarios, platform-specific regression gates, and retained CI evidence.
+
 ### Changed
 
 - Align the Typer support floor with the tested matrix and cover representative

docs/performance.md

@@ -1,42 +1,93 @@
 # Performance and adversarial-regression contract
 
 `base-cli` treats startup and filesystem behavior as part of its public
-quality contract. The checked benchmark is intentionally small and runs from
-the source checkout:
+quality contract. The benchmark is a comparative regression check, not a claim
+that a lifecycle framework should outpace bare parsers. Its scenarios separate
+interpreter/import cost, parser dispatch, the base-cli lifecycle, optional
+features, and persistence.
+
+Install the complete local validation set, including every comparator:
 
 ```bash
-python scripts/benchmark_runtime.py --check
+python -m pip install '.[dev,typer,quality,benchmark]'
+python scripts/benchmark_runtime.py --check --iterations 31 --output benchmark-results.json
 ```
 
-It records fresh-process import time and the cost of an isolated production
-invocation through `base_cli.testing.invoke`. The comparison mode measures
-equivalent no-op commands for base-cli, Click, Typer, and (when installed)
-Cyclopts. Install the optional benchmark extra to include Cyclopts:
+The benchmark is also part of the local aggregate:
 
 ```bash
-python -m pip install 'base-cli[benchmark]'
+./tests/full_validate.sh --gate benchmark
 ```
 
-The CI quality job checks the base-cli sample p95 against these budgets. The
-benchmark records the selected platform profile in both text and JSON output;
-set `BASE_CLI_BENCHMARK_PLATFORM` when a runner's filesystem or virtualization
-boundary is not represented by the host operating system. Supported profiles
-are `unix`, `macos`, `windows`, and `wsl`.
-
-| Measurement | Budget |
-| --- | ---: |
-| Fresh `import base_cli` (native Unix/macOS) | 750 ms |
-| Fresh `import base_cli` (native Windows) | 1,000 ms |
-| Fresh `import base_cli` (WSL2 on a Windows-mounted checkout) | 1,000 ms |
-| Isolated invocation and runtime filesystem setup | 1,500 ms |
-
-The benchmark reports the median, p95, and maximum for seven samples. Pass
-`--json` for a stable machine-readable result suitable for archiving or CI
-comparison. These
-budgets are intentionally broad enough for hosted runners while still
-detecting accidental quadratic startup work, unbounded metadata scans, or
-unexpected dependency imports. A performance improvement should preserve the
-same lifecycle and persistence assertions covered by the adversarial tests.
+## Scenario contract
+
+The comparative set is Click, Typer, Cyclopts, and base-cli. Each framework
+registers an equivalent zero-argument no-op command. Fresh-process
+measurements include Python startup, framework import, command construction,
+and dispatch through the framework's normal entry point. Warm parser samples
+reuse command objects; Click and Typer use Click's `CliRunner`, Cyclopts uses
+its `App` call, and base-cli reports both a shared Click-runner lifecycle
+sample and an end-to-end `base_cli.testing.invoke()` sample. The runner shape
+for each value is recorded here so comparisons do not imply identical
+mechanisms where framework APIs differ.
+
+Base-cli-only feature samples cover:
+
+- successful and failed JSON envelopes;
+- debug diagnostics on the user stream;
+- nested-command dispatch;
+- persistence disabled versus enabled, with the same log event in both cases.
+
+These feature costs are reported separately from parser comparisons. JSON
+success/error and nested dispatch use the public `App`/lifecycle API; persistence
+samples differ only in whether file logging is enabled. Measurements are
+in-process, warm, and use isolated temporary homes.
+
+## CI budgets and evidence
+
+CI collects 31 samples per scenario on Python 3.13 for each supported
+benchmark profile: native Unix, macOS, Windows, and WSL2. `--check` fails if a
+required comparator or scenario is missing, a p95 exceeds its profile budget,
+or the measured base-cli lifecycle increment over Click exceeds its separate
+profile budget. The lifecycle-to-Click ratio remains visible for interpretation,
+but is not itself gated because Click's sub-millisecond baseline makes ratios
+highly sensitive to timer granularity. Warm budgets apply to all non-persistence
+base-cli feature scenarios; file persistence has a separate platform budget
+because runner filesystems vary materially. Percentile gates catch practical
+regressions while keeping noisy single maxima visible without making one
+scheduler outlier block a change.
+
+| Budget (p95) | Unix | macOS | Windows | WSL2 |
+| --- | ---: | ---: | ---: | ---: |
+| Cold import, including interpreter startup | 750 ms | 750 ms | 1,000 ms | 1,000 ms |
+| Cold no-op invocation, including startup and dispatch | 2,000 ms | 2,000 ms | 4,000 ms | 4,000 ms |
+| Base-cli lifecycle increment over Click warm dispatch | 5 ms | 5 ms | 15 ms | 15 ms |
+| Warm invocation and non-persistence feature scenarios | 50 ms | 50 ms | 100 ms | 100 ms |
+| File-persistence-enabled scenario | 50 ms | 50 ms | 250 ms | 50 ms |
+
+An initial 31-sample local calibration on macOS (Python 3.14.6, Apple Silicon)
+measured approximately 101 ms for base-cli cold import, 0.56 ms for warm
+lifecycle dispatch, and 15.9 ms p95 for file-persisted logging. These are
+development-host measurements, not adoption claims or release comparisons.
+The first hosted 31-sample baseline measured file-persistence p95 at 22 ms on
+Ubuntu, 21 ms on macOS, 158 ms on Windows, and 27 ms on WSL2. Windows also had
+a high median absolute deviation (26 ms), so persistence has its own Windows
+budget instead of weakening other warm-scenario gates. These measurements are
+CI calibration evidence, not adoption claims or release comparisons; review
+subsequent retained artifacts before tightening platform budgets.
+
+Each report is versioned as `base-cli.benchmark` schema version 1 and contains
+the package version, source revision, UTC timestamp, platform profile, Python
+version/ABI, OS release, architecture, CPU count, sample count, medians, p95,
+maximum, median absolute deviation, and parser/lifecycle comparison values.
+The Tests workflow retains a distinct JSON artifact for each platform profile
+for 90 days. Download the artifact from the corresponding `Benchmark (...)`
+or `Validate (WSL)` Actions job to compare dated runs.
+
+The profile can be selected explicitly with
+`BASE_CLI_BENCHMARK_PLATFORM` when a runner's filesystem or virtualization
+boundary is not represented by its host OS. Supported values are `unix`,
+`macos`, `windows`, and `wsl`.
 
 ## Retention recovery work bounds
 

docs/testing.md

@@ -1,24 +1,50 @@
 # Validation commands
 
 The Base manifest declares `./tests/full_validate.sh` as the authoritative
-test command. It runs the repository baseline checks, Python tests with the
-coverage policy, strict typing, formatting and lint checks, schema and
-contract validation, documentation checks, compatibility-dashboard and
-performance checks, Bandit, and a strict `pip-audit` of the resolved
-third-party environment. Bandit and pip-audit are required; a missing tool is
-an error rather than a skipped check.
+local aggregate. It composes the same named gates used by CI: repository
+baseline, Python tests with coverage, strict typing, formatting and lint,
+documentation/schema/contract validation, benchmark budgets, and security.
+Bandit and pip-audit are required; a missing tool is an error rather than a
+skipped check.
 
-Run it from a clean checkout after installing the development and quality
-extras:
+Run it from a clean checkout after installing the development, quality, and
+benchmark extras (the latter installs every declared framework comparator):
 
 ```bash
-python -m pip install '.[dev,typer,quality]'
+python -m pip install '.[dev,typer,quality,benchmark]'
 ./tests/full_validate.sh
 ```
 
 `./tests/validate.sh` remains the fast repository-baseline check used when
 dependencies are not yet installed. It is not a substitute for the full
-validation gate. The full gate writes a machine-readable result to
+validation gate. Individual gates can be selected for focused local work:
+
+```bash
+./tests/full_validate.sh --gate runtime
+./tests/full_validate.sh --gate coverage
+./tests/full_validate.sh --gate typing
+./tests/full_validate.sh --gate style
+./tests/full_validate.sh --gate contracts
+./tests/full_validate.sh --gate benchmark
+./tests/full_validate.sh --gate security
+```
+
+The Tests workflow runs the runtime suite across the OS/Python matrix and on
+the supported Linux distributions/WSL. Its quality job runs platform-
+independent coverage, typing, style, contract, and security gates once, with
+each group visible as a named Actions step. A separate comparative benchmark
+matrix measures Click, Typer, Cyclopts, and base-cli on Unix, macOS, Windows,
+and WSL; each job publishes an Actions summary and retains its versioned JSON
+report as a dated artifact. The workflow validates feature branches through
+pull requests rather than launching a second full run on every feature-branch
+push; direct pushes to `main` and version tags remain validated. The Package
+workflow focuses on release-boundary checks: building and validating the
+wheel/sdist, checksums/SBOM, and clean installed-wheel smoke tests. It does
+not repeat the source test, typing, lint, documentation, benchmark, or
+security suites. `./tests/full_validate.sh` remains the one-command local
+aggregate of all source gates.
+
+The full gate writes a machine-readable result to
 `$BASE_CLI_VALIDATION_RESULT` (or `/tmp/base-cli-validation-result.json`). If
 Node.js is unavailable, the result is marked `partial`, the gate exits with
 status `2`, and it cannot be reported as an authoritative pass.