diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 5df179b0..fb34a407 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -3,6 +3,8 @@ version: 2 updates: - package-ecosystem: "cargo" directory: "/" + # Version PRs are consolidated by update-dependencies.yaml. Security updates remain enabled. + open-pull-requests-limit: 0 schedule: interval: "cron" cronjob: "0 5 2 * *" # Second day of each month at 05:00 UTC @@ -11,6 +13,8 @@ updates: - package-ecosystem: "github-actions" directory: "/" + # Version PRs are consolidated by update-dependencies.yaml. Security updates remain enabled. + open-pull-requests-limit: 0 schedule: interval: "cron" cronjob: "0 5 2 * *" # Second day of each month at 05:00 UTC @@ -19,6 +23,8 @@ updates: - package-ecosystem: "npm" directory: "/crates/string-offsets/js" + # Version PRs are consolidated by update-dependencies.yaml. Security updates remain enabled. + open-pull-requests-limit: 0 schedule: interval: "cron" cronjob: "0 5 2 * *" # Second day of each month at 05:00 UTC diff --git a/.github/scripts/apply-dependency-update b/.github/scripts/apply-dependency-update new file mode 100755 index 00000000..65671e7c --- /dev/null +++ b/.github/scripts/apply-dependency-update @@ -0,0 +1,187 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ecosystem=${1:?ecosystem is required} +bundle_dir=${2:?bundle directory is required} + +case "$ecosystem" in + cargo) + branch=automation/dependencies/cargo + deterministic_re='^crates/.*Cargo\.toml$' + agent_re='^crates/.*\.rs$' + final_re='^crates/.*(Cargo\.toml|\.rs)$' + ;; + npm) + branch=automation/dependencies/npm + deterministic_re='^crates/string-offsets/js/package(-lock)?\.json$' + agent_re='^crates/string-offsets/js/.*\.(js|cjs|mjs|ts)$' + final_re='^crates/string-offsets/js/(package(-lock)?\.json|.*\.(js|cjs|mjs|ts))$' + ;; + github-actions) + branch=automation/dependencies/github-actions + deterministic_re='^\.github/workflows/.*\.ya?ml$' + agent_re='a^' + final_re='^\.github/workflows/.*\.ya?ml$' + ;; + *) + echo "unsupported ecosystem: $ecosystem" >&2 + exit 2 + ;; +esac + +for path in state base.sha; do + if [[ ! -f "$bundle_dir/$path" ]]; then + echo "missing artifact file: $path" >&2 + exit 1 + fi +done + +expected_base=$(tr -d '[:space:]' < "$bundle_dir/base.sha") +.github/scripts/verify-dependency-base "$expected_base" >/dev/null + +state=$(tr -d '[:space:]' < "$bundle_dir/state") +case "$state" in + noop) + echo "No $ecosystem dependency changes; leaving branch and PR untouched." + exit 0 + ;; + ready) ;; + *) + echo "generator did not produce an applicable $ecosystem artifact" >&2 + exit 1 + ;; +esac + +for path in deterministic.patch final.patch title.txt body.md; do + if [[ ! -f "$bundle_dir/$path" ]]; then + echo "missing artifact file: $path" >&2 + exit 1 + fi +done + +tree_from_patch() { + local patch=$1 + local index + index=$(mktemp) + rm -f "$index" + GIT_INDEX_FILE=$index git read-tree HEAD + if [[ -s "$patch" ]]; then + GIT_INDEX_FILE=$index git apply --cached --binary "$patch" + fi + GIT_INDEX_FILE=$index git write-tree + rm -f "$index" +} + +validate_paths() { + local label=$1 + local regex=$2 + local paths=$3 + local violations + violations=$(grep -vE "$regex" "$paths" || true) + if [[ -n "$violations" ]]; then + echo "$label contains paths outside the allowlist:" >&2 + printf '%s\n' "$violations" >&2 + exit 1 + fi +} + +baseline_tree=$(tree_from_patch "$bundle_dir/deterministic.patch") +final_tree=$(tree_from_patch "$bundle_dir/final.patch") +git diff --name-only HEAD "$baseline_tree" > "$RUNNER_TEMP/deterministic-paths.txt" +git diff --name-only "$baseline_tree" "$final_tree" > "$RUNNER_TEMP/agent-paths.txt" +git diff --name-only HEAD "$final_tree" > "$RUNNER_TEMP/final-paths.txt" +git diff --diff-filter=AD --name-only HEAD "$final_tree" > "$RUNNER_TEMP/structural-paths.txt" + +validate_paths "deterministic update" "$deterministic_re" "$RUNNER_TEMP/deterministic-paths.txt" +validate_paths "agent update" "$agent_re" "$RUNNER_TEMP/agent-paths.txt" +validate_paths "final update" "$final_re" "$RUNNER_TEMP/final-paths.txt" + +if [[ -s "$RUNNER_TEMP/structural-paths.txt" ]]; then + echo "dependency update added or deleted files:" >&2 + cat "$RUNNER_TEMP/structural-paths.txt" >&2 + exit 1 +fi + +if [[ ! -s "$RUNNER_TEMP/final-paths.txt" ]]; then + echo "Final $ecosystem patch is empty; leaving branch and PR untouched." + exit 0 +fi + +git config user.name "github-actions[bot]" +git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + +expected_oid= +remote_oid=$(git ls-remote --heads origin "refs/heads/$branch" | awk '{print $1}') +if [[ -n "$remote_oid" ]]; then + git fetch origin "refs/heads/$branch:refs/remotes/origin/$branch" + expected_oid=$(git rev-parse "refs/remotes/origin/$branch") + non_bot=$(git log "origin/main..refs/remotes/origin/$branch" --format='%ae%x09%ce' | + awk -F '\t' '$1 != "41898282+github-actions[bot]@users.noreply.github.com" || $2 != "41898282+github-actions[bot]@users.noreply.github.com"') + if [[ -n "$non_bot" ]]; then + echo "$branch contains non-bot commits; refusing to overwrite it" >&2 + printf '%s\n' "$non_bot" >&2 + exit 1 + fi +fi + +prs=$(gh pr list \ + --head "$branch" \ + --state open \ + --limit 2 \ + --json number,isDraft,author,baseRefName) +count=$(jq 'length' <<<"$prs") +if [[ "$count" -gt 1 ]]; then + echo "multiple open PRs found for $branch" >&2 + exit 1 +fi + +if [[ "$count" -eq 1 ]]; then + number=$(jq -r '.[0].number' <<<"$prs") + author=$(jq -r '.[0].author.login' <<<"$prs") + draft=$(jq -r '.[0].isDraft' <<<"$prs") + base=$(jq -r '.[0].baseRefName' <<<"$prs") + if [[ "$author" != "github-actions[bot]" || "$draft" != "true" || "$base" != "main" ]]; then + echo "open PR for $branch is not the workflow's own main-targeting draft" >&2 + exit 1 + fi +fi + +if [[ $(wc -l < "$bundle_dir/title.txt") -ne 1 || $(wc -c < "$bundle_dir/title.txt") -gt 200 ]]; then + echo "invalid PR title" >&2 + exit 1 +fi +title=$(tr -d '\n' < "$bundle_dir/title.txt") + +cat "$bundle_dir/body.md" > "$RUNNER_TEMP/pr-body.md" + +git checkout -B "$branch" refs/remotes/origin/main +git apply --index --binary "$bundle_dir/final.patch" +git commit --no-verify \ + -m "$title" \ + -m "Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>" + +if [[ -n "$expected_oid" ]]; then + git push --force-with-lease="refs/heads/$branch:$expected_oid" origin "HEAD:refs/heads/$branch" +else + git push --force-with-lease="refs/heads/$branch:" origin "HEAD:refs/heads/$branch" +fi + +if [[ "$count" -eq 1 ]]; then + jq -n \ + --arg title "$title" \ + --rawfile body "$RUNNER_TEMP/pr-body.md" \ + '{title: $title, body: $body}' | + gh api -X PATCH "repos/$GITHUB_REPOSITORY/pulls/$number" --input - >/dev/null + echo "Updated draft PR #$number." +else + gh pr create \ + --base main \ + --head "$branch" \ + --draft \ + --title "$title" \ + --body-file "$RUNNER_TEMP/pr-body.md" +fi + +gh api -X POST "repos/$GITHUB_REPOSITORY/actions/workflows/ci.yaml/dispatches" -f ref="$branch" +echo "Dispatched CI for $branch." diff --git a/.github/scripts/generate-dependency-update b/.github/scripts/generate-dependency-update new file mode 100755 index 00000000..d18cf21e --- /dev/null +++ b/.github/scripts/generate-dependency-update @@ -0,0 +1,263 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ecosystem=${1:?ecosystem is required} +bundle_dir=${2:?bundle directory is required} + +if [[ -n "${NPM_TOKEN:-}" || -n "${NODE_AUTH_TOKEN:-}" ]]; then + echo "dependency generation refuses npm publish credentials" >&2 + exit 1 +fi + +agent_output_dir=.dependency-agent-output +agent_context_dir=${RUNNER_TEMP:-/tmp}/dependency-agent-context +candidate_title=$agent_context_dir/candidate-title.txt +candidate_body=$agent_context_dir/candidate-body.md + +mkdir -p "$bundle_dir" "$agent_output_dir" "$agent_context_dir" +rm -f \ + "$bundle_dir/base.sha" \ + "$bundle_dir/deterministic.patch" \ + "$bundle_dir/final.patch" \ + "$bundle_dir/state" \ + "$bundle_dir/title.txt" \ + "$bundle_dir/body.md" \ + "$agent_output_dir/title.txt" \ + "$agent_output_dir/body.md" \ + "$candidate_title" \ + "$candidate_body" + +printf 'failure\n' > "$bundle_dir/state" + +case "$ecosystem" in + cargo|npm|github-actions) ;; + *) + echo "unsupported ecosystem: $ecosystem" >&2 + exit 2 + ;; +esac + +run_updater() { + case "$ecosystem" in + cargo) + cargo upgrade --incompatible allow --pinned allow --recursive true + rm -f Cargo.lock + ;; + npm) + .github/scripts/update-npm-dependencies + ;; + github-actions) + pinact run --update --min-age 14 --branch-to-tag '^main$' --branch-to-tag '^master$' + ;; + esac +} + +run_validation() { + local log=$1 + local rc + + set +e + set -o pipefail + { + case "$ecosystem" in + cargo) + rm -f Cargo.lock + make lint && make test && make build + ;; + npm) + rm -rf crates/string-offsets/js/node_modules crates/string-offsets/js/pkg + npm --prefix crates/string-offsets/js ci --ignore-scripts + make lint && make test && make build && make build-js + ;; + github-actions) + mapfile -t workflows < <(find .github/workflows -maxdepth 1 -type f \( -name '*.yaml' -o -name '*.yml' \) -print) + ruby -rpsych -e 'ARGV.each { |path| Psych.parse_file(path) }' "${workflows[@]}" + pinact run --fix=false --no-api + ;; + esac + } 2>&1 | tee "$log" + rc=${PIPESTATUS[0]} + set -e + + return "$rc" +} + +tree_from_patch() { + local patch=$1 + local index + index=$(mktemp) + rm -f "$index" + GIT_INDEX_FILE=$index git read-tree HEAD + if [[ -s "$patch" ]]; then + GIT_INDEX_FILE=$index git apply --cached --binary "$patch" + fi + GIT_INDEX_FILE=$index git write-tree + rm -f "$index" +} + +validate_agent_paths() { + local baseline_tree=$1 + local final_tree=$2 + local structural_changes + local violations + + git diff --name-only "$baseline_tree" "$final_tree" > "$agent_context_dir/agent-paths.txt" + structural_changes=$(git diff --diff-filter=AD --name-only "$baseline_tree" "$final_tree") + if [[ -n "$structural_changes" ]]; then + echo "agent added or deleted files:" >&2 + printf '%s\n' "$structural_changes" >&2 + return 1 + fi + + case "$ecosystem" in + cargo) + violations=$(grep -vE '^crates/.*\.rs$' "$agent_context_dir/agent-paths.txt" || true) + ;; + npm) + violations=$(grep -vE '^crates/string-offsets/js/.*\.(js|cjs|mjs|ts)$' "$agent_context_dir/agent-paths.txt" || true) + ;; + github-actions) + violations=$(cat "$agent_context_dir/agent-paths.txt") + ;; + esac + + if [[ -n "$violations" ]]; then + echo "agent edited paths outside the $ecosystem allowlist:" >&2 + printf '%s\n' "$violations" >&2 + return 1 + fi +} + +run_agent() { + local attempt=$1 + local prompt + + mkdir -p "$agent_output_dir" + rm -f "$agent_output_dir/title.txt" "$agent_output_dir/body.md" + + prompt=$(cat < "$bundle_dir/base.sha" + +echo "Running deterministic $ecosystem updater" +run_updater 2>&1 | tee "$agent_context_dir/updater.log" + +if git diff --quiet; then + printf 'noop\n' > "$bundle_dir/state" + echo "No tracked $ecosystem dependency changes." + exit 0 +fi + +run_validation "$agent_context_dir/validation.log" || true +git diff --binary --full-index --no-ext-diff > "$bundle_dir/deterministic.patch" +cp "$bundle_dir/deterministic.patch" "$agent_context_dir/deterministic.diff" +deterministic_hash=$(sha256sum "$bundle_dir/deterministic.patch" | awk '{print $1}') +baseline_tree=$(tree_from_patch "$bundle_dir/deterministic.patch") + +max_attempts=3 +if [[ "$ecosystem" == "github-actions" ]]; then + max_attempts=1 +fi + +final_validation_rc=1 +for attempt in $(seq 1 "$max_attempts"); do + if ! run_agent "$attempt" 2>&1 | tee "$agent_context_dir/agent-$attempt.log"; then + echo "Copilot CLI failed on attempt $attempt." >&2 + exit 1 + fi + + if [[ ! -s "$agent_output_dir/title.txt" || ! -s "$agent_output_dir/body.md" ]]; then + echo "agent did not produce a title and body" >&2 + exit 1 + fi + cp "$agent_output_dir/title.txt" "$candidate_title" + cp "$agent_output_dir/body.md" "$candidate_body" + rm -f "$agent_output_dir/title.txt" "$agent_output_dir/body.md" + if ! rmdir "$agent_output_dir"; then + echo "agent left unexpected files in $agent_output_dir" >&2 + exit 1 + fi + + untracked=$(git ls-files --others --exclude-standard) + if [[ -n "$untracked" ]]; then + echo "agent created untracked files:" >&2 + printf '%s\n' "$untracked" >&2 + exit 1 + fi + + current_hash=$(sha256sum "$bundle_dir/deterministic.patch" | awk '{print $1}') + if [[ "$current_hash" != "$deterministic_hash" ]]; then + echo "deterministic snapshot changed while the agent was running" >&2 + exit 1 + fi + + git diff --binary --full-index --no-ext-diff > "$bundle_dir/final.patch" + final_tree=$(tree_from_patch "$bundle_dir/final.patch") + validate_agent_paths "$baseline_tree" "$final_tree" + + if run_validation "$agent_context_dir/validation.log"; then + final_validation_rc=0 + break + fi +done + +if [[ "$final_validation_rc" -ne 0 ]]; then + echo "machine validation still fails after $max_attempts agent attempt(s)" >&2 + exit 1 +fi + +if [[ $(wc -l < "$candidate_title") -ne 1 || $(wc -c < "$candidate_title") -gt 200 ]]; then + echo "agent title must be one line and at most 200 bytes" >&2 + exit 1 +fi + +if [[ $(wc -c < "$candidate_body") -gt 50000 ]]; then + echo "agent body exceeds 50 KB" >&2 + exit 1 +fi + +git diff --binary --full-index --no-ext-diff > "$bundle_dir/final.patch" +cp "$candidate_title" "$bundle_dir/title.txt" +cp "$candidate_body" "$bundle_dir/body.md" +printf 'ready\n' > "$bundle_dir/state" diff --git a/.github/scripts/test-dependency-automation b/.github/scripts/test-dependency-automation new file mode 100755 index 00000000..81b8aaac --- /dev/null +++ b/.github/scripts/test-dependency-automation @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root=$(git rev-parse --show-toplevel) +workflow=$root/.github/workflows/update-dependencies.yaml +generate=$root/.github/scripts/generate-dependency-update +apply=$root/.github/scripts/apply-dependency-update +verify_base=$root/.github/scripts/verify-dependency-base +update_npm=$root/.github/scripts/update-npm-dependencies +test_root=$root/target/dependency-automation-tests + +rm -rf "$test_root" +mkdir -p "$test_root" +trap 'rm -rf "$test_root"' EXIT + +ruby -rpsych -e 'Psych.parse_file(ARGV.fetch(0))' "$workflow" +bash -n "$generate" "$apply" "$verify_base" "$update_npm" + +if grep -Eq '^[[:space:]]+(NPM_TOKEN|NODE_AUTH_TOKEN):' "$workflow"; then + echo "dependency workflow exposes npm publish credentials" >&2 + exit 1 +fi +if grep -Fq 'secrets.NPM_TOKEN' "$workflow"; then + echo "dependency workflow reads the npm publish secret" >&2 + exit 1 +fi +if grep -Eq -- '--secret-env-vars=.*(NPM_TOKEN|NODE_AUTH_TOKEN)' "$generate"; then + echo "Copilot receives npm publish credentials" >&2 + exit 1 +fi +for token in NPM_TOKEN NODE_AUTH_TOKEN; do + if env "$token=test" "$generate" npm "$test_root/token-bundle" >/dev/null 2>&1; then + echo "dependency generation accepted $token" >&2 + exit 1 + fi +done + +[[ $(grep -Fc "if: github.ref == 'refs/heads/main'" "$workflow") -eq 1 ]] +grep -Fq "if: always() && github.ref == 'refs/heads/main'" "$workflow" +[[ $(grep -Fc 'ref: main' "$workflow") -eq 2 ]] +grep -Fq 'verify-dependency-base' "$generate" +grep -Fq "verify-dependency-base \"\$expected_base\"" "$apply" +grep -Fq "git checkout -B \"\$branch\" refs/remotes/origin/main" "$apply" + +git_fixture=$test_root/git +git init --bare "$git_fixture/origin.git" >/dev/null +git init -b main "$git_fixture/work" >/dev/null +git -C "$git_fixture/work" config user.name test +git -C "$git_fixture/work" config user.email test@example.com +printf 'main\n' > "$git_fixture/work/base" +git -C "$git_fixture/work" add base +git -C "$git_fixture/work" commit -m main >/dev/null +git -C "$git_fixture/work" remote add origin "$git_fixture/origin.git" +git -C "$git_fixture/work" push -u origin main >/dev/null +main_oid=$(git -C "$git_fixture/work" rev-parse HEAD) +[[ $(cd "$git_fixture/work" && "$verify_base") == "$main_oid" ]] + +git -C "$git_fixture/work" switch -c feature >/dev/null +printf 'feature\n' >> "$git_fixture/work/base" +git -C "$git_fixture/work" commit -am feature >/dev/null +if (cd "$git_fixture/work" && "$verify_base") >/dev/null 2>&1; then + echo "feature ref passed the origin/main base check" >&2 + exit 1 +fi +git -C "$git_fixture/work" switch main >/dev/null +if (cd "$git_fixture/work" && "$verify_base" "${main_oid}0") >/dev/null 2>&1; then + echo "stale dependency artifact passed the origin/main base check" >&2 + exit 1 +fi + +stale_noop=$test_root/stale-noop +mkdir -p "$stale_noop" "$git_fixture/work/.github/scripts" +printf 'noop\n' > "$stale_noop/state" +printf '%s\n' "$main_oid" > "$stale_noop/base.sha" +cp "$apply" "$verify_base" "$git_fixture/work/.github/scripts/" +printf 'advanced\n' >> "$git_fixture/work/base" +git -C "$git_fixture/work" commit -am advanced >/dev/null +git -C "$git_fixture/work" push origin main >/dev/null +if (cd "$git_fixture/work" && .github/scripts/apply-dependency-update npm "$stale_noop") \ + >"$test_root/stale-noop.log" 2>&1; then + echo "stale no-op artifact was accepted after origin/main advanced" >&2 + exit 1 +fi +grep -Fq 'dependency artifact was generated from a different origin/main' \ + "$test_root/stale-noop.log" + +npm_fixture=$test_root/npm +mkdir -p "$npm_fixture" +cat > "$npm_fixture/package.json" <<'EOF' +{ + "name": "dependency-automation-major-test", + "version": "1.0.0", + "private": true, + "devDependencies": { + "jest": "^29.0.0" + } +} +EOF +env \ + -u npm_command \ + -u npm_config_local_prefix \ + -u npm_config_user_agent \ + -u npm_execpath \ + -u npm_lifecycle_event \ + -u npm_lifecycle_script \ + -u npm_node_execpath \ + -u npm_package_json \ + -u npm_package_name \ + -u npm_package_version \ + NPM_CHECK_UPDATES_VERSION=19.1.1 "$update_npm" "$npm_fixture" +node - "$npm_fixture/package.json" "$npm_fixture/package-lock.json" <<'NODE' +const fs = require("fs"); +const packageJson = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); +const lock = JSON.parse(fs.readFileSync(process.argv[3], "utf8")); +const range = packageJson.devDependencies.jest; +const match = /^\^(\d+)\./.exec(range); + +if (!match || Number(match[1]) <= 29) { + throw new Error(`expected npm updater to cross the Jest major boundary, got ${range}`); +} +if (lock.packages[""].devDependencies.jest !== range) { + throw new Error("package-lock.json did not preserve the updated direct range"); +} +NODE diff --git a/.github/scripts/update-npm-dependencies b/.github/scripts/update-npm-dependencies new file mode 100755 index 00000000..2f4a3ea5 --- /dev/null +++ b/.github/scripts/update-npm-dependencies @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +package_dir=${1:-crates/string-offsets/js} +: "${NPM_CHECK_UPDATES_VERSION:?NPM_CHECK_UPDATES_VERSION is required}" + +( + cd "$package_dir" + npm exec --yes --package="npm-check-updates@$NPM_CHECK_UPDATES_VERSION" -- \ + npm-check-updates --upgrade --target latest + npm install --package-lock-only --ignore-scripts --no-audit --no-fund +) diff --git a/.github/scripts/verify-dependency-base b/.github/scripts/verify-dependency-base new file mode 100755 index 00000000..6b41f97b --- /dev/null +++ b/.github/scripts/verify-dependency-base @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -euo pipefail + +expected_base=${1:-} +head_oid=$(git rev-parse HEAD) +main_oid=$(git rev-parse refs/remotes/origin/main) + +if [[ "$head_oid" != "$main_oid" ]]; then + echo "dependency automation must run from the current origin/main" >&2 + echo "HEAD: $head_oid" >&2 + echo "origin/main: $main_oid" >&2 + exit 1 +fi + +if [[ -n "$expected_base" && "$expected_base" != "$main_oid" ]]; then + echo "dependency artifact was generated from a different origin/main" >&2 + echo "artifact: $expected_base" >&2 + echo "origin/main: $main_oid" >&2 + exit 1 +fi + +printf '%s\n' "$main_oid" diff --git a/.github/skills/update-deps/SKILL.md b/.github/skills/update-deps/SKILL.md index 807626e2..eaf7fc6b 100644 --- a/.github/skills/update-deps/SKILL.md +++ b/.github/skills/update-deps/SKILL.md @@ -1,289 +1,84 @@ --- name: update-deps -description: Keep dependencies up-to-date. Discovers outdated deps via dependabot alerts/PRs, creates one PR per ecosystem, iterates until CI is green, then assigns for review. +description: Keep Cargo, npm, and GitHub Actions dependencies current through the repository's guarded dependency automation. user-invocable: true --- # Update Dependencies -Automate the full dependency update lifecycle: discover what's outdated, apply updates grouped by ecosystem, fix breakage, get CI green, and hand off for human review. +Maintain one stable draft pull request per ecosystem without giving the Copilot CLI GitHub write credentials. -## Repository context +## Repository inventory -This is a Rust workspace containing utility crates published to crates.io. All dependency update PRs target the **`main`** branch. +The workspace has 14 Cargo manifests: the virtual root, eight published crates (`bpe`, `bpe-openai`, `casefold`, `consistent-choose-k`, `geo_filters`, `hash-sorted-map`, `sparse-ngrams`, and `string-offsets`), and five benchmark/test support packages. -Dependabot is configured (`.github/dependabot.yaml`) to open PRs against `main` on the 2nd of each month. This skill gathers individual dependabot PRs, combines updates by ecosystem, fixes any breakage, gets CI green, and creates consolidated PRs for human review. +`Cargo.lock` is intentionally ignored and is not a durable dependency record. Cargo updates must change dependency requirements in the workspace manifests with the pinned `cargo-edit` version used by `.github/workflows/update-dependencies.yaml`; `cargo update` alone is not an update. -### Crates in this workspace - -| Crate | Description | +| Ecosystem | Durable files | |---|---| -| **bpe** | Fast byte-pair encoding | -| **bpe-openai** | OpenAI tokenizers built on bpe | -| **geo_filters** | Probabilistic cardinality estimation | -| **string-offsets** | UTF-8/UTF-16/Unicode position conversion (with WASM/JS bindings) | - -Supporting packages (not published): `bpe-tests`, `bpe-benchmarks`. - -### Ecosystems in this repo - -| Ecosystem | Directories | Notes | -|---|---|---| -| **cargo** | `/` (workspace root) | Deps declared per-crate; `Cargo.lock` at workspace root pins versions | -| **github-actions** | `.github/workflows/` | CI and publish workflows | -| **npm** | `crates/string-offsets/js/` | JS bindings for string-offsets (WASM) | - -### Build and validation commands - -```bash -make build # cargo build --all-targets --all-features -make build-js # npm run compile in crates/string-offsets/js -make lint # cargo fmt --check + cargo clippy (deny warnings, forbid unwrap_used) -make test # cargo test + doc tests -``` - -CI runs on `ubuntu-latest` with the `mold` linker. The lint job depends on build. - -## Workflow - -### 1. Assess repo state - -Determine the repo identity and confirm the target branch. - -```bash -git remote get-url origin # extract owner/repo -git fetch origin main -git rev-parse --verify origin/main -``` +| Cargo | `crates/**/Cargo.toml` | +| npm | `crates/string-offsets/js/package.json`, `crates/string-offsets/js/package-lock.json` | +| GitHub Actions | `.github/workflows/*.yaml`, `.github/workflows/*.yml` | -Detect which ecosystems have pending updates: +## Automation contract -```bash -[ -f Cargo.toml ] && echo "cargo" -ls .github/workflows/*.yml .github/workflows/*.yaml 2>/dev/null && echo "github-actions" -[ -f crates/string-offsets/js/package.json ] && echo "npm" -``` +`.github/workflows/update-dependencies.yaml` runs at the Thursday 06:17 UTC fleet slot. Its matrix is deliberately serial in this order: Cargo, npm, GitHub Actions. -Report discovered ecosystems to the user. +Roll out this repository-local automation before removing any existing dependency-update coverage, so there is no gap. -### 2. Gather dependency intelligence +Each ecosystem run has two trust domains: -Fetch open dependabot PRs: +1. The read-only `generate` job runs a pinned updater, captures its deterministic patch, invokes a checksum-verified Copilot CLI release, runs machine validation, and uploads immutable patch and metadata artifacts. +2. The `apply` job never executes agent output. It validates both patches, checks deterministic and agent path allowlists independently, refuses artifacts generated from any commit other than its current `origin/main`, refuses non-bot history on the reserved branch, applies the final patch as data, and uses force-with-lease only on that reserved branch. -```bash -gh pr list --author 'app/dependabot' --base main --state open --json number,title,headRefName,labels --limit 100 -``` +Manual dispatches are accepted only from `main`. Both jobs explicitly check out `main`, and the trusted scripts require `HEAD` to equal `refs/remotes/origin/main`. -Fetch open dependabot alerts: +The Copilot CLI is started with `--add-dir .` so this project skill is loaded as trusted local configuration. It receives no shell or GitHub MCP tool, cannot ask questions, and has only read/search/edit/web tools. It may repair consumer code, assess risk, and write the proposed PR title/body to a temporary in-tree handoff directory that the trusted script removes before snapshotting. It must not run git, push, create PRs, edit dependency manifests, or edit workflows. -```bash -gh api --paginate /repos/{owner}/{repo}/dependabot/alerts --jq '[.[] | select(.state=="open") | {number: .number, package: .security_vulnerability.package.name, ecosystem: .security_vulnerability.package.ecosystem, severity: .security_advisory.severity, summary: .security_advisory.summary}]' -``` - -For ecosystems without dependabot coverage or when running ad-hoc, use native tooling: +## Ecosystem behavior -- **cargo:** `cargo update --dry-run` -- **npm:** find directories containing `package.json`, then run `npm outdated --json || true` in each (npm exits non-zero when updates exist) +### Cargo -Also fetch the advisory URLs for any security-related updates. Individual alert details are at `https://github.com/{owner}/{repo}/security/dependabot/{alert_number}`. Fetch alert numbers and GHSA IDs via: +- Run the pinned `cargo-edit` release with compatible, incompatible, pinned, and recursive upgrades enabled. +- Delete the ignored generated `Cargo.lock`; only manifest changes are durable. +- The agent may repair Rust source files under `crates/**`. +- Validate with `make lint`, `make test`, and `make build`. -```bash -gh api --paginate /repos/{owner}/{repo}/dependabot/alerts --jq '[.[] | {number: .number, state, package: .security_vulnerability.package.name, ecosystem: .security_vulnerability.package.ecosystem, severity: .security_advisory.severity, ghsa_id: .security_advisory.ghsa_id, summary: .security_advisory.summary}]' -``` +### npm -Include both open and auto_dismissed/dismissed alerts — the update may resolve alerts in any state. +- Run the pinned `npm-check-updates` release with `--target latest` in `crates/string-offsets/js`, then regenerate `package-lock.json` with lifecycle scripts disabled. Direct ranges must move across major versions. +- Preserve and commit `package-lock.json`. +- Dependency updates, installs, builds, tests, and lifecycle code run without `NPM_TOKEN` or `NODE_AUTH_TOKEN`; the package uses only the public registry. +- Install the workflow's pinned `wasm-pack` before validation; never rely on the Makefile's unpinned fallback installation. +- The agent may repair tracked JavaScript consumer/test files under `crates/string-offsets/js/**`, excluding `package.json` and `package-lock.json`. +- Validate with `make lint`, `make test`, `make build`, and `make build-js`. -Cross-reference and group all updates by ecosystem. Present a summary to the user: +### GitHub Actions -- How many updates per ecosystem -- Which have security alerts (with severity, GHSA IDs, and advisory links) -- Which dependabot PRs already exist +- Run the pinned `pinact` release with `--update` and a 14-day minimum release age. +- Keep every action reference pinned to a full commit SHA with its version annotation. +- Agent edits are forbidden; the agent only summarizes the deterministic update. +- Validate workflow YAML parsing and offline SHA pinning with `pinact --fix=false --no-api`. -**Flag high-risk upgrades.** Before proceeding, explicitly call out upgrades that carry elevated risk: +## Pull request behavior -- **Major version bumps** — likely contain breaking API changes -- **Packages with wide blast radius** — for this repo, pay special attention to: `serde`, `itertools`, `regex-automata`, `wasm-bindgen`, `criterion`, and the Rust toolchain itself -- **Multiple major bumps in the same PR** — each major bump multiplies the risk; consider splitting them +- Branches are stable: `automation/dependencies/cargo`, `automation/dependencies/npm`, and `automation/dependencies/github-actions`. +- Reuse the workflow's own open draft PR for the branch. Refuse a non-draft PR, a PR by another author, a different base, multiple open PRs, or any non-bot commit on the reserved branch. +- A clean diff is a successful no-op: do not push, create, close, or edit a PR. +- Never mark a PR ready, merge it, close superseded PRs, or request review. `CODEOWNERS` routes changes to `@github/blackbird-reviewers`. +- After creating or updating a draft PR, explicitly dispatch `ci.yaml` on the reserved branch. This is nonblocking and does not depend on recursive workflow events. +- Repair is bounded to three agent passes. Missing output, allowlist violations, failed final validation, and unexpected branch/PR state are explicit failures. -Present the risk assessment to the user and recommend which upgrades to include vs. defer. When in doubt, prefer a smaller, safe update over an ambitious one that might break. +## Authentication -### 3. Create branch and apply updates +The read-only generator uses `GITHUB_TOKEN` with `copilot-requests: write`, which bills Copilot usage to the organization. GitHub write authentication belongs only to the separate trusted apply job and must never be exposed to the Copilot CLI. -For each selected ecosystem, starting from `main`: +## Validation commands ```bash -git checkout main -git pull origin main -git checkout -b deps/{ecosystem}-updates-$(date +%Y-%m-%d) +make lint +make test +make build +make build-js # npm/WASM changes +.github/scripts/test-dependency-automation ``` - -Apply updates using ecosystem-appropriate tooling: - -**cargo:** - -```bash -cargo update -# For major bumps, edit Cargo.toml version constraints then: -cargo check -``` - -This is a Cargo workspace — always run from the repo root. All crate `Cargo.toml` files are in `crates/`. The `Cargo.lock` at the root is the single source of truth. - -**npm:** - -```bash -cd crates/string-offsets/js -npm update -npm install -``` - -**github-actions:** - -- Parse workflow YAML files in `.github/workflows/` for `uses:` directives -- For each action with an outdated version (from dependabot PRs/alerts), update the SHA or version tag -- Be careful to preserve comments and formatting - -### 4. Build, lint, and test locally - -Always run: - -```bash -make lint # cargo fmt --check + clippy with deny warnings -make test # cargo test with backtrace -make build # full workspace build (all targets, all features) -``` - -If npm dependencies changed: - -```bash -make build-js # npm compile for string-offsets JS binding -``` - -**If the build/lint/test fails:** - -1. Read the error output carefully -2. Analyze what broke — likely API changes, type errors, or deprecation removals -3. Make the necessary code changes to fix the breakage -4. Run the pipeline again -5. Repeat up to 3 times - -If still failing after 3 iterations, report the situation to the user and ask for guidance. Do not push broken code. - -### 5. Commit and push - -Stage all changes and commit with a descriptive message: - -```bash -git add -A -git commit -m "chore(deps): update {ecosystem} dependencies - -Updated packages: -- package-a: 1.0.0 → 2.0.0 -- package-b: 3.1.0 → 3.2.0 - -{If code changes were needed:} -Fixed breaking changes: -- Updated X API usage for package-a v2 - -Supersedes: #{dependabot_pr_1}, #{dependabot_pr_2} -" -``` - -Push the branch: - -```bash -git push -u origin HEAD -``` - -### 6. Create the PR - -**Title:** `chore(deps): update {ecosystem} dependencies` - -**Body should include:** - -- List of updated dependencies with version changes (old → new) -- Any security alerts resolved — for each, link to the specific dependabot alert (`https://github.com/{owner}/{repo}/security/dependabot/{alert_number}`) and the GHSA advisory (`https://github.com/advisories/GHSA-xxxx-xxxx-xxxx`), along with severity and summary -- **High-risk changes flagged for reviewer attention** (major version bumps, wide-blast-radius packages) -- Code changes made to fix breakage (if any) -- References to superseded dependabot PRs -- Note that this was generated by the update-deps skill - -Write the body to a temp file and create the PR **targeting `main`**: - -```bash -gh pr create --title "chore(deps): update {ecosystem} dependencies" --body-file /tmp/deps-pr-body.md --base main -rm /tmp/deps-pr-body.md -``` - -### 7. Monitor CI and iterate on failures - -Watch the PR's checks: - -```bash -gh pr checks {pr_number} --watch --fail-fast -``` - -**If checks fail:** - -1. Get the failed run details: - -```bash -gh run list --branch {branch} --status failure --json databaseId,name --limit 1 -gh run view {run_id} --log-failed -``` - -2. Analyze the failure — CI runs on `ubuntu-latest` with `mold` linker, which may differ from local builds. - -3. Fix the issue locally, commit, and push: - -```bash -git add -A -git commit -m "fix: resolve CI failure in {ecosystem} dep update - -{Brief description of what failed and why}" -git push -``` - -4. Monitor again. Repeat up to 3 iterations total. - -5. If still failing after 3 pushes, report to the user with the failure details and ask for help. - -### 8. Close superseded dependabot PRs - -For each dependabot PR that this update supersedes: - -```bash -gh pr close {dependabot_pr_number} --comment "Superseded by #{new_pr_number} which includes this update along with other {ecosystem} dependency updates." -``` - -### 9. Assign for review - -Request review from CODEOWNERS or a user-provided reviewer (not the PR author): - -```bash -gh pr edit {pr_number} --add-reviewer {reviewer_login} -``` - -Report the final PR URL and a summary of what was done. - -## Guidelines - -- **All PRs target `main`.** There is no separate dev branch. -- **Never push to `main` directly.** Always work on a feature branch. -- **Never push code that doesn't pass `make lint` and `make test`.** If you can't fix it in 3 tries, stop and ask. -- **Be conservative with major version bumps.** If a major version update breaks things and the fix isn't obvious, skip that package and note it in the PR description. -- **Regenerate lockfiles.** Always regenerate `Cargo.lock` and `package-lock.json` after updating — don't just edit manifests. -- **One ecosystem at a time.** Complete the full cycle (update → build → push → PR → CI green) for one ecosystem before moving to the next. -- **If no updates are needed** for an ecosystem, skip it and tell the user. -- **Security alerts take priority.** Address security alerts first within each ecosystem. -- **Clippy is strict.** This repo forbids `unwrap_used` outside tests and denies all warnings. New dependency versions may trigger new clippy lints — fix them. - -## Edge cases - -- **Cargo workspace:** Dependencies are declared per-crate but share a single `Cargo.lock` at the workspace root. Always run `cargo update` and `cargo check` from the repo root. -- **npm:** Look for `package.json` files to discover npm packages rather than hardcoding paths — the repo layout may change. -- **WASM builds:** After updating `wasm-bindgen` or related deps, verify `make build-js` still works — WASM toolchain version mismatches are common. -- **Rate limits:** If `gh api` hits rate limits, wait and retry. Report to user if persistent. -- **Nothing to update:** Report cleanly and move to the next ecosystem (or exit). -- **Merge conflicts on push:** Rebase on `main` and retry: `git fetch origin main && git rebase origin/main`. -- **Branch already exists:** If `deps/{ecosystem}-updates-{date}` already exists, append a counter or ask user. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6ddf0a13..5104592d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,9 +21,9 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 + - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v4.4.0 - name: Build run: make build @@ -36,9 +36,12 @@ jobs: runs-on: ubuntu-latest needs: build steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 + - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v4.4.0 + + - name: Test dependency automation + run: .github/scripts/test-dependency-automation - name: Check formatting and clippy run: make lint @@ -47,9 +50,9 @@ jobs: name: Test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 + - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v4.4.0 - name: Run unit tests run: make test diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 51388d4f..61853f2a 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -14,8 +14,8 @@ jobs: run: working-directory: crates/string-offsets/js steps: - - uses: actions/checkout@v7 - - uses: actions/setup-node@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 registry-url: https://registry.npmjs.org/ diff --git a/.github/workflows/update-dependencies.yaml b/.github/workflows/update-dependencies.yaml new file mode 100644 index 00000000..c3d3fa75 --- /dev/null +++ b/.github/workflows/update-dependencies.yaml @@ -0,0 +1,151 @@ +name: Update dependencies + +on: + schedule: + - cron: "17 6 * * 4" + workflow_dispatch: + +permissions: {} + +concurrency: + group: update-dependencies + cancel-in-progress: false + +jobs: + generate: + name: Generate ${{ matrix.ecosystem }} update + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 90 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + # Keep the shared Thursday 06:17 UTC fleet slot, but stagger expensive work internally. + ecosystem: [cargo, npm, github-actions] + permissions: + contents: read + copilot-requests: write + env: + GITHUB_TOKEN: ${{ github.token }} + COPILOT_VERSION: 1.0.83-3 + COPILOT_SHA256: 868c68d92d207cf1b68b4791b94b38d2cdeefc0717986bc5cd71843a83b41603 + CARGO_EDIT_VERSION: 0.13.10 + NPM_CHECK_UPDATES_VERSION: 19.1.1 + WASM_PACK_VERSION: 0.15.0 + PINACT_VERSION: 4.1.1 + PINACT_SHA256: d1cffebe5704b74e2e5f8a864efb9f7e54768972dc686188c008033fb1797841 + steps: + - name: Checkout main without write credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + ref: main + + - name: Install pinned Copilot CLI + run: | + set -euo pipefail + archive="$RUNNER_TEMP/copilot-linux-x64.tar.gz" + curl -fsSL --retry 3 \ + "https://github.com/github/copilot-cli/releases/download/v${COPILOT_VERSION}/copilot-linux-x64.tar.gz" \ + -o "$archive" + echo "${COPILOT_SHA256} ${archive}" | sha256sum --check - + mkdir -p "$RUNNER_TEMP/copilot" + tar -xzf "$archive" -C "$RUNNER_TEMP/copilot" + echo "$RUNNER_TEMP/copilot" >> "$GITHUB_PATH" + + - name: Install Rust tools + if: matrix.ecosystem == 'cargo' || matrix.ecosystem == 'npm' + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component clippy,rustfmt + rustup default stable + if [[ "${{ matrix.ecosystem }}" == "cargo" ]]; then + cargo install cargo-edit --locked --version "$CARGO_EDIT_VERSION" + else + cargo install wasm-pack --locked --version "$WASM_PACK_VERSION" + fi + + - name: Setup Node + if: matrix.ecosystem == 'npm' + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + cache: npm + cache-dependency-path: crates/string-offsets/js/package-lock.json + + - name: Install pinned pinact + if: matrix.ecosystem == 'github-actions' + run: | + set -euo pipefail + archive="$RUNNER_TEMP/pinact_linux_amd64.tar.gz" + curl -fsSL --retry 3 \ + "https://github.com/suzuki-shunsuke/pinact/releases/download/v${PINACT_VERSION}/pinact_linux_amd64.tar.gz" \ + -o "$archive" + echo "${PINACT_SHA256} ${archive}" | sha256sum --check - + mkdir -p "$RUNNER_TEMP/pinact" + tar -xzf "$archive" -C "$RUNNER_TEMP/pinact" + echo "$RUNNER_TEMP/pinact" >> "$GITHUB_PATH" + + - name: Generate immutable update artifact + id: generate + env: + BUNDLE_DIR: ${{ runner.temp }}/dependency-update/${{ matrix.ecosystem }} + run: | + set +e + .github/scripts/generate-dependency-update "${{ matrix.ecosystem }}" "$BUNDLE_DIR" + rc=$? + echo "rc=$rc" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Upload immutable update artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: dependency-update-${{ matrix.ecosystem }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/dependency-update/${{ matrix.ecosystem }} + retention-days: 7 + if-no-files-found: error + + - name: Report explicit generation failure + if: steps.generate.outputs.rc != '0' + env: + GENERATE_RC: ${{ steps.generate.outputs.rc }} + run: | + echo "::error::${{ matrix.ecosystem }} dependency generation failed with exit ${GENERATE_RC}." + exit "$GENERATE_RC" + + apply: + name: Apply ${{ matrix.ecosystem }} update + needs: generate + if: always() && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + ecosystem: [cargo, npm, github-actions] + permissions: + actions: write + contents: write + pull-requests: write + env: + GITHUB_TOKEN: ${{ github.token }} + steps: + - name: Checkout main for guarded apply + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + ref: main + + - name: Download immutable update artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: dependency-update-${{ matrix.ecosystem }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/dependency-update/${{ matrix.ecosystem }} + + - name: Apply guarded update and create or reuse draft PR + env: + BUNDLE_DIR: ${{ runner.temp }}/dependency-update/${{ matrix.ecosystem }} + run: .github/scripts/apply-dependency-update "${{ matrix.ecosystem }}" "$BUNDLE_DIR"