diff --git a/.github/workflows/stickydisk-gc.yml b/.github/workflows/stickydisk-gc.yml new file mode 100644 index 00000000000..2305c7dc434 --- /dev/null +++ b/.github/workflows/stickydisk-gc.yml @@ -0,0 +1,147 @@ +name: Sticky Disk GC + +# node_modules sticky disks are keyed on hashFiles('bun.lock') by design — see the +# "Mount node_modules" comment in test-build.yml. A sticky disk is a mutable volume +# and `bun install --frozen-lockfile` adds what the lockfile needs without pruning +# what it dropped, so branches on different lockfiles must not share one. That +# design is correct and is preserved here; its cost is a new 6-16 GB disk per +# lockfile bump, created at ~4.7/day. +# +# Blacksmith already evicts any sticky disk after 7 days of inactivity, so this is +# not a leak — it is a retention window that is far too generous for this key. The +# median disk is only USED for ~0.16 days and then billed for another 7, so the +# retention tail is almost the entire cost. This collects at 3 days instead. +# +# Age-based on purpose. The key contains NO PR identifier, so every open PR whose +# checkout has the same bun.lock mounts the SAME disk — with ~180 open PRs sharing +# on the order of 20 distinct lockfile hashes, the shared case is the common case. +# Deleting on PR close would therefore destroy a disk that dozens of other open PRs +# are actively using. Never add a pull_request or pull_request_target trigger here. + +on: + schedule: + - cron: '17 9 * * *' + workflow_dispatch: + inputs: + retention_days: + description: Delete node_modules disks unused for more than this many days. + required: false + default: '3' + dry_run: + description: List what would be deleted without deleting it. + type: boolean + required: false + default: false + +# Nothing in this job reads the repository. +permissions: {} + +concurrency: + group: stickydisk-gc + cancel-in-progress: false + +jobs: + gc: + name: Reclaim idle node_modules sticky disks + # GitHub-hosted on purpose, not a Blacksmith runner: the CLI is a pure API + # client, and GC has to keep working during a Blacksmith outage or a + # CI_PROVIDER break-glass switch — exactly when disks sit idle and still bill. + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + # Pinned binary + checksum rather than `curl https://get.blacksmith.sh | sh`: + # this job holds an org-wide token, so it must not execute unpinned remote + # shell. Bump both values together; the vendor publishes a .sha256 sidecar + # next to the binary to check against. + BLACKSMITH_CLI_VERSION: v0.4.57 + BLACKSMITH_CLI_SHA256: 7f60f3b9f8d4d7644d9743f5d962acb3b3dbf675f51676702e5f292e02060bca + # The CLI self-updates in the background on every invocation, which would + # silently defeat the pin above. + BLACKSMITH_DISABLE_AUTO_UPDATE: '1' + BLACKSMITH_ORG: simstudioai + TARGET_REPO: ${{ github.repository }} + RETENTION_DAYS: ${{ inputs.retention_days || '3' }} + DRY_RUN: ${{ inputs.dry_run || 'false' }} + + steps: + - name: Install Blacksmith CLI + run: | + set -euo pipefail + url="https://clireleases.blacksmith.sh/cli/${BLACKSMITH_CLI_VERSION}/linux/amd64/blacksmith" + curl -fsSL "$url" -o /usr/local/bin/blacksmith + echo "${BLACKSMITH_CLI_SHA256} /usr/local/bin/blacksmith" | sha256sum -c - + chmod +x /usr/local/bin/blacksmith + + - name: Authenticate + env: + BLACKSMITH_CLI_TOKEN: ${{ secrets.BLACKSMITH_CLI_TOKEN }} + run: | + set -euo pipefail + printf '%s' "$BLACKSMITH_CLI_TOKEN" \ + | blacksmith auth login --api-token - --non-interactive --organization "$BLACKSMITH_ORG" + + - name: Delete node_modules disks idle beyond the retention window + run: | + set -euo pipefail + + blacksmith stickydisk list \ + --repo "$TARGET_REPO" \ + --search '-node-modules-' \ + --per-page 100 \ + --format json > disks.json + + # Two independent guards, because the blast radius of a wrong key is a + # cache every CI job depends on: + # 1. --search narrows server-side to the node_modules family. + # 2. The regex re-proves each key's full shape locally. The event + # segment is [a-z_]+ rather than an enumerated list — the key + # interpolates ${{ github.event_name }}, and a workflow_dispatch + # disk already exists that an enumerated push|pull_request would + # have silently skipped forever. + # Verified against the live account: this matches all 39 node_modules + # disks and none of the 19 bun/turbo/Docker disks, which are mounted + # every run, never idle, and must survive. + # + # Grouped by key before the staleness test because `delete` without + # --arch removes every architecture variant, so a key may only go when + # its NEWEST variant is stale. + jq -r --arg repo "$TARGET_REPO" --argjson days "$RETENTION_DAYS" ' + (now - ($days * 86400)) as $cutoff + | .entries + | map(select(.type == "stickydisk")) + | map(select(.key | test("^" + ($repo | gsub("/"; "\\/")) + "-node-modules-[a-z_]+-[0-9a-f]{64}$"))) + | group_by(.key) + | map({ + key: .[0].key, + gb: (map(.size_bytes) | add / 1000000000 * 100 | round / 100), + last_used: (map(.last_used_at | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601) | max) + }) + | map(select(.last_used < $cutoff)) + | .[] | "\(.key)\t\(.gb)" + ' disks.json > stale.tsv + + count=$(wc -l < stale.tsv | tr -d ' ') + reclaimed=$(awk -F'\t' '{s+=$2} END {printf "%.1f", s+0}' stale.tsv) + { + echo "### Sticky disk GC" + echo "Retention: ${RETENTION_DAYS}d · dry run: ${DRY_RUN} · candidates: ${count} (${reclaimed} GB)" + } >> "$GITHUB_STEP_SUMMARY" + + failed=0 + while IFS=$'\t' read -r key gb; do + [ -n "$key" ] || continue + if [ "$DRY_RUN" = "true" ]; then + echo "- would delete \`${key}\` (${gb} GB)" >> "$GITHUB_STEP_SUMMARY" + continue + fi + if blacksmith stickydisk delete --repo "$TARGET_REPO" --key "$key" --yes; then + echo "- deleted \`${key}\` (${gb} GB)" >> "$GITHUB_STEP_SUMMARY" + else + echo "- FAILED \`${key}\`" >> "$GITHUB_STEP_SUMMARY" + failed=1 + fi + done < stale.tsv + + # Fail loudly rather than continue-on-error: a revoked token or a changed + # CLI JSON shape would otherwise silently revert us to 7-day billing. + exit "$failed"