From df0885c1def894eadfd8d9358d859d1a611d24dc Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:18:39 +0200 Subject: [PATCH 1/4] feat(security): an origin gate for the app container, shipped off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The API's gate (#11208) closed one of two doors and said so in its own docstring: `anyplot-app` stands with `ingress=all` beside it, serving the whole site from a `*.run.app` URL with no bot challenge, no WAF and no rate limit, and relaying any crawler user agent through `@seo_proxy` into a repository query and an outbound Plausible event. `app/origin-gate.conf.template` is the nginx half of the same mechanism: the base image's own envsubst entrypoint renders the maps with the shared secret before nginx starts, and every server block refuses what the Cloudflare edge did not stamp. `ORIGIN_GATE` unset means off, `on` means 403, and armed with no secret fails CLOSED — the map keys are tagged so an empty value cannot become "match anything". Nothing is armed by merging. `/_health` reports `X-Origin-Gate` with the API's five verdicts, so every route into the container can be measured while the gate is still off; the rollout, the hostnames the Transform Rule has to cover and the rollback are in infra/cloudflare/README.md. The three callers that reach this origin without the edge now carry the header themselves: the pre-traffic smoke, the daily bot monitor, and the apex Worker's `/api/event` branch — the one path that Worker sends to this container, and one a Worker subrequest leaves unstamped. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UEScQMZFvxxNNyNJYryfa3 --- .github/workflows/bot-serving-check.yml | 73 +++++- .github/workflows/ci-image.yml | 200 ++++++++++++++++ app/Dockerfile | 25 ++ app/cloudbuild.yaml | 43 +++- app/nginx.conf | 112 ++++++++- app/origin-gate.conf.template | 119 ++++++++++ changelog.d/app-origin-gate.md | 49 ++++ docs/reference/api.md | 22 +- docs/reference/seo.md | 23 ++ infra/cloudflare/README.md | 172 +++++++++++++- infra/cloudflare/anyplot-api-proxy.js | 20 +- tests/unit/api/test_app_origin_gate.py | 296 ++++++++++++++++++++++++ 12 files changed, 1129 insertions(+), 25 deletions(-) create mode 100644 app/origin-gate.conf.template create mode 100644 changelog.d/app-origin-gate.md create mode 100644 tests/unit/api/test_app_origin_gate.py diff --git a/.github/workflows/bot-serving-check.yml b/.github/workflows/bot-serving-check.yml index af71f8ce8f7..f1b73389468 100644 --- a/.github/workflows/bot-serving-check.yml +++ b/.github/workflows/bot-serving-check.yml @@ -15,6 +15,13 @@ # that broke in the incident above; Cloudflare-edge issues are out of this # monitor's reach by design. # +# Skipping the edge is exactly what the app's origin gate refuses +# (app/origin-gate.conf.template), so every probe below carries the header the +# edge would have stamped, out of the ORIGIN_SECRET repository secret. This +# monitor is the reason the gate could not be a Host rule: it cannot spoof the +# Host either, and anything else it might present as an exception would be +# public with this repository. +# # What the bot pages are is DERIVED from the repo, never written out here. The # routes come from the `@router.get("/seo-proxy/…")` decorators in # api/routers/seo.py and the spec title from plots//specification.yaml. @@ -56,14 +63,15 @@ jobs: # 36 check() calls (10 derived bot routes + spec page + impl page + # ClaudeBot + 15 crawler UAs + 404 + robots + sitemap + 3 llms + 2 human # controls) x (--retry 2 -> up to 3 attempts x --max-time 30) can reach - # ~54 min worst-case, plus five non-retried probes (llms.txt charset, + # ~54 min worst-case, plus the retried /_health gate probe (90s) and five + # non-retried probes (llms.txt charset, # trailing slash, og-image, .well-known redirect, the /{spec}/{language} - # 301 — 30s each); 62 leaves + # 301 — 30s each); 64 leaves # room to report a clean failure rather than dying to the job timeout, # which reports nothing useful. Recompute this when adding checks: the # ceiling is check() calls x 90s, plus margin. The route sweep grows with # api/routers/seo.py, so a new bot page adds 90s to that ceiling. - timeout-minutes: 62 + timeout-minutes: 64 steps: # The routes and the expected title are read out of the repo, so it has # to be here before the first request goes out. @@ -73,6 +81,19 @@ jobs: fetch-depth: 1 - name: Crawler UAs must get 200 + per-route pages + env: + # The header the Cloudflare Transform Rule stamps on everything it + # proxies for anyplot.ai. These checks target the Cloud Run ORIGIN on + # purpose (see the file header: Cloudflare 403s runner IPs even for a + # UA-spoofed Googlebot), so they skip the edge and have to stamp their + # own — the same repository secret sync-postgres.yml sends to the API + # gate, and the same value both services are given. It reaches curl + # through the environment and this step runs without `set -x`, so it + # is never echoed. An exception keyed on anything else — a header this + # workflow invents, a user agent — would be public with this + # repository, which is why the shared secret is the only exception the + # gate can afford. + ORIGIN_SECRET: ${{ secrets.ORIGIN_SECRET }} run: | set -uo pipefail # Cloud Run origin of the anyplot-app service (see header comment @@ -85,12 +106,46 @@ jobs: HUMAN="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36" fail=0 + # Absent secret => no header at all. Sending an empty one instead + # would read as a mismatch, i.e. as a half-applied rotation rather + # than as the missing secret it is. + HDR=() + if [ -n "${ORIGIN_SECRET}" ]; then HDR=(-H "X-Origin-Secret: ${ORIGIN_SECRET}"); fi + + # Ask the gate before anything else. /_health is its one exempt path, + # so it answers whatever the gate is doing, and it reports the verdict + # for the request it was asked with — never the value. Without this, + # an armed gate plus a missing repository secret would red all ~36 + # checks below with 403s and open an incident that reads "every + # crawler page is broken" — the wrong thing to be woken up for, and + # the wrong thing to go looking for. + gate=$(curl -sS --retry 2 --max-time 30 -o /dev/null -D - "${HDR[@]}" "$ORIGIN/_health" \ + | tr -d '\r' | sed -n 's/^[Xx]-[Oo]rigin-[Gg]ate: //p') || gate="" + case "$gate" in + ok|off-seen) + echo "origin gate: $gate — the probes below carry the header" ;; + off) + echo "::warning::the origin gate is not armed AND this run sent no X-Origin-Secret. Set the ORIGIN_SECRET repository secret to the value the anyplot-app service will be given, before it is armed." ;; + missing) + echo "::error::the origin gate is armed and this run has no ORIGIN_SECRET repository secret — every check below would be a 403. Set it to the same value as the ORIGIN_SECRET on the anyplot-app Cloud Run service." + exit 1 ;; + mismatch) + echo "::error::the origin gate is armed and the ORIGIN_SECRET repository secret is not the value the anyplot-app service was given — a half-applied rotation. Roll the gate back, finish the rotation, arm again." + exit 1 ;; + *) + # No header at all: the running revision predates the gate. That + # is deploy lag, not a serving fault, and every check below still + # means exactly what it always meant — so it is said out loud and + # the run continues. + echo "::warning::$ORIGIN/_health carries no X-Origin-Gate header, so the deployed revision predates the origin gate. Check for a pending deploy." ;; + esac + check() { local ua="$1" url="$2" expect="$3" want="${4:-200}" local code # On curl failure REPLACE the code — a failing curl can still have # printed a partial -w code; appending would yield e.g. "200000". - code=$(curl -sS --retry 2 --max-time 30 -A "$ua" -o body.html -w '%{http_code}' "$url") || code="000" + code=$(curl -sS --retry 2 --max-time 30 "${HDR[@]}" -A "$ua" -o body.html -w '%{http_code}' "$url") || code="000" if [ "$code" != "$want" ]; then echo "::error::$url with UA '$ua' returned HTTP $code (expected $want)" fail=1 @@ -250,7 +305,7 @@ jobs: # to the same hub would consolidate nothing, and the endpoint documents # a permanent redirect. read -r lang_code lang_target <<< "$(curl -sS --max-time 30 -o /dev/null \ - -A "$GOOGLEBOT" -w '%{http_code} %{redirect_url}' "$ORIGIN/$SPEC/python")" + "${HDR[@]}" -A "$GOOGLEBOT" -w '%{http_code} %{redirect_url}' "$ORIGIN/$SPEC/python")" if [ "$lang_code" != "301" ]; then echo "::error::/$SPEC/python answered HTTP $lang_code (expected a permanent 301 onto the hub)" fail=1 @@ -312,7 +367,7 @@ jobs: # The site card must be the FILE for a preview bot, not the proxy — # a preview bot that lands on /seo-proxy/og-image.png shows nothing. - code=$(curl -sS --max-time 30 -A "$TWITTERBOT" -o /dev/null -w '%{http_code} %{content_type}' "$ORIGIN/og-image.png") || code="000" + code=$(curl -sS --max-time 30 "${HDR[@]}" -A "$TWITTERBOT" -o /dev/null -w '%{http_code} %{content_type}' "$ORIGIN/og-image.png") || code="000" case "$code" in "200 image/png"*) echo "OK: og-image.png served as image to a preview bot" ;; *) echo "::error::og-image.png for a preview bot: $code (expected 200 image/png)"; fail=1 ;; @@ -320,7 +375,7 @@ jobs: # A guessed /.well-known/llms.txt must land on the file, not on the # SPA shell (which soft-404'd it with 200 until 2026-08-28). - wk_target=$(curl -sS --max-time 30 -o /dev/null -A "$CHATGPTUSER" \ + wk_target=$(curl -sS --max-time 30 -o /dev/null "${HDR[@]}" -A "$CHATGPTUSER" \ -w '%{redirect_url}' "$ORIGIN/.well-known/llms.txt") case "$wk_target" in */llms.txt) echo "OK: .well-known/llms.txt -> $wk_target" ;; @@ -340,7 +395,7 @@ jobs: # llms.txt carries UTF-8 punctuation (em dashes, arrows); without an # explicit charset a strict client decodes it as Latin-1 mojibake. - ct=$(curl -sS --max-time 30 -A "$GOOGLEBOT" -o /dev/null -w '%{content_type}' "$ORIGIN/llms.txt") + ct=$(curl -sS --max-time 30 "${HDR[@]}" -A "$GOOGLEBOT" -o /dev/null -w '%{content_type}' "$ORIGIN/llms.txt") case "$ct" in *charset=utf-8*) echo "OK: llms.txt content-type: $ct" ;; *) @@ -351,7 +406,7 @@ jobs: # A trailing slash must normalise to the canonical URL on THIS host. # It used to 307 to http://api.anyplot.ai/seo-proxy/... — internal # path, wrong host, plain http, and that host disallows all crawling. - slash_target=$(curl -sS --max-time 30 -o /dev/null -A "$GOOGLEBOT" \ + slash_target=$(curl -sS --max-time 30 -o /dev/null "${HDR[@]}" -A "$GOOGLEBOT" \ -w '%{redirect_url}' "$ORIGIN/$SPEC/") case "$slash_target" in "") diff --git a/.github/workflows/ci-image.yml b/.github/workflows/ci-image.yml index 42c699ee543..c83849b7c63 100644 --- a/.github/workflows/ci-image.yml +++ b/.github/workflows/ci-image.yml @@ -255,3 +255,203 @@ jobs: - name: Skip notice if: steps.check.outputs.should_build == 'false' run: echo "::notice::Image build skipped - no changes to api/, core/, the dependency lock or a Dockerfile" + + # The same idea for the frontend container, and for a sharper reason: what + # app/Dockerfile produces is not a program that fails to import, it is an + # nginx that either boots or does not — and since the origin gate + # (app/origin-gate.conf.template) the config it boots with is RENDERED at + # container start from two environment variables. Nothing before this job + # ever ran that entrypoint: the deploy's pre-traffic smoke is the first place + # the rendered config exists, and it lives in Cloud Build, after the merge. + # + # A separate job rather than more steps in the one above, so an SPA build + # never delays the API smoke and neither failure hides the other. Its change + # detection is deliberately narrow: `app/src/**` cannot affect any of this, + # and the frontend opens far too many PRs to pay for a container build each + # time. The event-shape handling is the one above in short form — the long + # rationale for each branch is written out there. + app-image: + name: Build app image and smoke the origin gate + runs-on: ubuntu-latest + permissions: + contents: read + # yarn install + vite build cold is minutes; the gate matrix below is + # seconds. 20 fails loudly instead of hanging on a slow package index. + timeout-minutes: 20 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Check for app-container changes + id: check + env: + EVENT_NAME: ${{ github.event_name }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + MG_BASE_SHA: ${{ github.event.merge_group.base_sha }} + MG_HEAD_SHA: ${{ github.event.merge_group.head_sha }} + PUSH_BEFORE: ${{ github.event.before }} + PUSH_AFTER: ${{ github.event.after }} + FORCE_RUN: ${{ inputs.force_run }} + run: | + set -uo pipefail + CHANGED_FILES="" + case "$EVENT_NAME" in + pull_request) + if ! CHANGED_FILES=$(git diff --name-only "$PR_BASE_SHA" "$PR_HEAD_SHA"); then + echo "::error::could not diff $PR_BASE_SHA..$PR_HEAD_SHA — refusing to decide whether the app image needs building" + exit 1 + fi ;; + merge_group) + if ! CHANGED_FILES=$(git diff --name-only "$MG_BASE_SHA" "$MG_HEAD_SHA"); then + echo "::error::could not diff $MG_BASE_SHA..$MG_HEAD_SHA — refusing to decide whether the app image needs building" + exit 1 + fi ;; + push) + if [[ "$PUSH_BEFORE" =~ ^0+$ ]] || ! CHANGED_FILES=$(git diff --name-only "$PUSH_BEFORE" "$PUSH_AFTER"); then + echo "::warning::the pushed range is not diffable — building the app image rather than skipping the gate" + echo "should_build=true" >> "$GITHUB_OUTPUT" + exit 0 + fi ;; + *) + if ! CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD); then + echo "::warning::no parent commit to diff against — building the app image rather than skipping the gate" + echo "should_build=true" >> "$GITHUB_OUTPUT" + exit 0 + fi ;; + esac + + echo "Changed files:" + echo "$CHANGED_FILES" + + # What can change how this container boots or what it serves: the + # Dockerfile, the three nginx files, the build inputs, and what the + # build context can see. Not app/src — the gate is upstream of every + # line of it. + APP_CHANGES=$(echo "$CHANGED_FILES" | grep -E '^(app/(Dockerfile|nginx\.conf|security-headers\.conf|origin-gate\.conf\.template|package\.json|yarn\.lock|vite\.config\.ts|index\.html|\.dockerignore)$|\.github/workflows/ci-image\.yml$)' || true) + + if [[ "$EVENT_NAME" == "workflow_dispatch" && "$FORCE_RUN" == "true" ]]; then + echo "Manual trigger with force_run=true, will build the app image" + echo "should_build=true" >> "$GITHUB_OUTPUT" + elif [[ -n "$APP_CHANGES" ]]; then + echo "Found app-container changes, will build the app image" + echo "should_build=true" >> "$GITHUB_OUTPUT" + else + echo "No app-container changes, skipping the build" + echo "should_build=false" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Buildx + if: steps.check.outputs.should_build == 'true' + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + + # Its own cache scope: this image shares no layer with the API's, and one + # shared scope would have them evicting each other. + - name: Build the app image + if: steps.check.outputs.should_build == 'true' + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: app + file: app/Dockerfile + push: false + load: true + tags: anyplot-app:ci + build-args: | + VITE_API_URL=https://api.anyplot.ai + VITE_DEBUG_API_URL=/api + cache-from: type=gha,scope=app-image + cache-to: type=gha,mode=max,scope=app-image + + # THE assert this job exists for: the gate's three states, against the + # real image, through the real entrypoint. A test over the config text can + # only say what the file says; only a running container can say that + # envsubst rendered it, that nginx accepted the result, and that the maps + # decide what they were meant to decide. + - name: Origin gate matrix (off / armed / armed with no secret) + if: steps.check.outputs.should_build == 'true' + run: | + set -euo pipefail + # Not a production value and never one: this container is thrown away + # at the end of the job, and the real secret has no business on a + # runner that prints its own logs on failure. + SECRET="ci-origin-gate-not-a-production-secret" + + start() { # start [docker run args...] + local name="$1" port="$2"; shift 2 + docker run -d --name "$name" -p "$port:8080" "$@" anyplot-app:ci >/dev/null + for _ in $(seq 30); do + if curl -fsS "localhost:$port/_health" >/dev/null 2>&1; then return 0; fi + sleep 1 + done + echo "::error::container $name never answered /_health within 30 s" + docker logs "$name" || true + exit 1 + } + code() { curl -s -o /dev/null -w '%{http_code}' "$@"; } + verdict() { curl -sI "$@" | tr -d '\r' | sed -n 's/^[Xx]-[Oo]rigin-[Gg]ate: //p'; } + want() { # want + if [ "$2" != "$3" ]; then echo "::error::$1: got '$2', expected '$3'"; exit 1; fi + echo "OK: $1 = $3" + } + + # 1. Unset — the state this change ships in and rolls back to. Nothing + # is refused, and /_health already reports whether a header arrived, + # which is what makes arming a measurement instead of a leap. + start gate-off 8081 + want "gate off: the shell still serves" "$(code localhost:8081/)" 200 + want "gate off: no header" "$(verdict localhost:8081/_health)" off + want "gate off: a header arrived" "$(verdict -H 'X-Origin-Secret: anything' localhost:8081/_health)" off-seen + + # 2. Armed. The door is shut, the exempt path is not, and the refusal + # says which of the two armed failures it is. + start gate-on 8082 -e ORIGIN_GATE=on -e ORIGIN_SECRET="$SECRET" + want "armed: no header is refused" "$(code localhost:8082/)" 403 + want "armed: wrong secret is refused" "$(code -H "X-Origin-Secret: wrong" localhost:8082/)" 403 + want "armed: the right secret passes" "$(code -H "X-Origin-Secret: $SECRET" localhost:8082/)" 200 + want "armed: /_health stays exempt" "$(code localhost:8082/_health)" 200 + want "armed: /_health names the failure" "$(verdict localhost:8082/_health)" missing + want "armed: a wrong secret is a mismatch" "$(verdict -H "X-Origin-Secret: wrong" localhost:8082/_health)" mismatch + want "armed: the right secret is ok" "$(verdict -H "X-Origin-Secret: $SECRET" localhost:8082/_health)" ok + # The refusal is a page, not nginx's stock one — which would print the + # exact nginx version to anyone knocking on the raw origin. + curl -s localhost:8082/ -o denied.html + grep -qF "answers only requests that came through the edge" denied.html \ + || { echo "::error::the 403 body is not the gate's own page"; cat denied.html; exit 1; } + # The one thing that must never leak, in the two places it could. + # `if !` rather than `grep … && exit`: under `set -e` a grep that + # finds nothing — the passing case — would end the script itself. + if grep -qF "$SECRET" denied.html; then + echo "::error::the 403 body carries the secret"; exit 1 + fi + if docker logs gate-on 2>&1 | grep -qF "$SECRET"; then + echo "::error::the container logged the secret"; exit 1 + fi + echo "OK: the secret is in neither the refusal nor the logs" + # The rendered config has to be valid nginx, and `nginx -t` says so + # without dumping it — `nginx -T` would print the secret into this log. + docker exec gate-on nginx -t + echo "OK: the rendered config validates" + + # 3. Armed with NO secret must fail CLOSED. This is the whole reason + # the map keys are tagged: an untagged `"${ORIGIN_SECRET}"` key + # would render as `""`, which is exactly what an absent header + # looks like — and a forgotten variable would open the door to the + # entire internet while looking armed. + start gate-shut 8083 -e ORIGIN_GATE=on + want "armed, no secret: refused" "$(code localhost:8083/)" 403 + want "armed, no secret: empty header too" "$(code -H 'X-Origin-Secret;' localhost:8083/)" 403 + want "armed, no secret: /_health exempt" "$(code localhost:8083/_health)" 200 + + - name: Container logs on failure + if: failure() && steps.check.outputs.should_build == 'true' + run: | + for c in gate-off gate-on gate-shut; do + echo "=== $c ===" + docker logs "$c" 2>&1 || true + done + + - name: Skip notice + if: steps.check.outputs.should_build == 'false' + run: echo "::notice::App image build skipped - no changes to app/Dockerfile, the nginx configuration or the build inputs" diff --git a/app/Dockerfile b/app/Dockerfile index c7d0af3e166..3a59df70974 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -33,6 +33,31 @@ FROM nginxinc/nginx-unprivileged:alpine COPY --chown=nginx:nginx nginx.conf /etc/nginx/conf.d/default.conf COPY --chown=nginx:nginx security-headers.conf /etc/nginx/security-headers.conf +# The origin gate's maps, which need the shared secret — and nginx cannot read +# the environment. The base image already ships the official entrypoint's +# 20-envsubst-on-templates.sh, which renders /etc/nginx/templates/*.template +# into /etc/nginx/conf.d/ before nginx starts, so this costs no start script of +# our own. The `00-` prefix puts the rendered maps ahead of default.conf in the +# conf.d glob, where a reader expects to meet them. +COPY --chown=nginx:nginx origin-gate.conf.template /etc/nginx/templates/00-origin-gate.conf.template + +# Two defaults that have to exist as real ENVIRONMENT variables rather than as a +# documented meaning of "unset". envsubst substitutes only names it finds in the +# environment: with neither set, `${ORIGIN_SECRET}` would survive into the +# rendered config as a literal, nginx would read it as a reference to a variable +# it does not know, and the container would refuse to start. With them set, a +# service that declares neither means "gate off, no secret" — which is the +# default state this change ships in and the rollback it rolls back to. +# +# NGINX_ENVSUBST_FILTER keeps envsubst away from everything else: without it, +# envsubst rewrites every `$name` in the template whose name happens to exist in +# the environment, and `$host` and `$uri` in there are nginx's variables, not the +# shell's. The filter is a regex over variable NAMES; `^ORIGIN_` is exactly +# ORIGIN_GATE and ORIGIN_SECRET. +ENV ORIGIN_GATE=off \ + ORIGIN_SECRET="" \ + NGINX_ENVSUBST_FILTER="^ORIGIN_" + # Copy built assets from builder stage COPY --from=builder --chown=nginx:nginx /app/dist /usr/share/nginx/html diff --git a/app/cloudbuild.yaml b/app/cloudbuild.yaml index f271493d962..4e820ffbfb1 100644 --- a/app/cloudbuild.yaml +++ b/app/cloudbuild.yaml @@ -141,8 +141,45 @@ steps: # candidate is ALWAYS cold, and the crawler probes additionally wait on # the first upstream call to api.anyplot.ai. RETRY="--retry 5 --retry-delay 5 --retry-all-errors" + # The candidate is probed on its `run.app` tag URL, which by definition + # never passes the Cloudflare edge — so once ORIGIN_GATE=on is set on the + # service (app/origin-gate.conf.template) every probe but /_health needs + # the header the edge would have stamped. Read here rather than through + # `availableSecrets` on purpose, exactly as api/cloudbuild.yaml does: + # that resolves at build start and would fail every build until the + # secret exists. Missing secret or missing permission => empty => the + # probes run bare, which is correct while the gate is off and is caught + # by the /_health verdict below once it is on. The value is captured, + # never echoed; the step runs without `set -x`. + ORIGIN_SECRET=$$(gcloud secrets versions access latest --secret=ORIGIN_SECRET 2>/dev/null || true) + HDR=() + if [ -n "$$ORIGIN_SECRET" ]; then HDR=(-H "X-Origin-Secret: $$ORIGIN_SECRET"); fi HUMAN="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36" GOOGLEBOT="Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" + # Ask the gate what it makes of this build BEFORE any content probe, so + # a wired-up-wrong secret is reported as itself rather than as a + # mystifying 403 on the home page. /_health is the gate's one exempt + # path, so it answers whatever the gate is doing, and it reports the + # verdict for the request it was asked with — never the value. + # `off`/`off-seen` are ACCEPTED, not failures: they are the gate before + # it is armed and after a rollback, and a build that refused to run then + # would break the deploy pipeline exactly when it is needed most. An + # ABSENT header is a failure of its own: it means the templated maps + # never rendered, which is the one way this mechanism can fail quietly. + # + # Fetched to a file rather than piped: `-ceu` carries no `pipefail`, so a + # curl that never reached the candidate at all would leave `$$gate` empty + # and be reported below as a config that failed to render — a wrong + # answer, on the log someone reads while a deploy is blocked. + curl -fsS $$RETRY "$${HDR[@]}" -o /dev/null -D gate.out "$$URL/_health" \ + || { echo "the candidate never answered /_health"; exit 1; } + gate=$$(tr -d '\r' < gate.out | sed -n 's/^[Xx]-[Oo]rigin-[Gg]ate: //p') + case "$$gate" in + ok|off|off-seen) echo "origin gate: $$gate" ;; + missing) echo "the candidate's gate is armed and this build sent no secret — every probe below would be a 403"; exit 1 ;; + mismatch) echo "the candidate's gate is armed and this build's secret is not the one it was given"; exit 1 ;; + *) echo "/_health carries no X-Origin-Gate header ('$$gate') — the templated origin-gate maps did not render"; exit 1 ;; + esac # `expect `: fetch to a file rather # than piping into `grep -q`. `grep -q` exits at the first match and # SIGPIPEs curl, so the pipe form only ever passes because `-ceu` @@ -151,7 +188,7 @@ steps: # probe failed and what was expected, which is the whole value of a log # someone reads while a deploy is blocked. expect() { - curl -fsS $$RETRY -A "$$1" -o body.out "$$URL$$2" \ + curl -fsS $$RETRY "$${HDR[@]}" -A "$$1" -o body.out "$$URL$$2" \ || { echo "candidate did not serve $$2 to UA '$$1'"; exit 1; } grep -qF "$$3" body.out || { echo "$$4 ($$2 is missing: $$3)"; exit 1; } echo "OK: $$2" @@ -176,7 +213,7 @@ steps: # while every browser got the untouched file (Copilot review). # Reproduced against a local nginx with an index.html.gz planted in the # docroot: plain curl saw 7 stamped tags, `curl --compressed` saw 0. - curl -fsS $$RETRY --compressed -A "$$HUMAN" -D head.out -o body.out "$$URL/" \ + curl -fsS $$RETRY "$${HDR[@]}" --compressed -A "$$HUMAN" -D head.out -o body.out "$$URL/" \ || { echo "candidate did not serve the shell for the nonce probe"; exit 1; } # Exactly 32 hex digits, which is what nginx's $$request_id always is. # `[0-9a-f]*` would have accepted a bare `nonce-` — and `test -n` calls @@ -213,7 +250,7 @@ steps: "robots.txt did not come from the location = bypass" expect "$$GOOGLEBOT" "/llms.txt" '# anyplot' \ "llms.txt did not come from the location = bypass" - ct=$$(curl -fsS $$RETRY -A "$$GOOGLEBOT" -o /dev/null -w '%{content_type}' "$$URL/llms.txt") + ct=$$(curl -fsS $$RETRY "$${HDR[@]}" -A "$$GOOGLEBOT" -o /dev/null -w '%{content_type}' "$$URL/llms.txt") case "$$ct" in *charset=utf-8*) ;; *) echo "llms.txt served as '$$ct' (expected a utf-8 charset)"; exit 1 ;; diff --git a/app/nginx.conf b/app/nginx.conf index 3706e44497f..58b343de98a 100644 --- a/app/nginx.conf +++ b/app/nginx.conf @@ -145,6 +145,30 @@ server { listen 8080 default_server; server_name _; + # Origin gate. The verdict and every variable behind it come from + # app/origin-gate.conf.template, which the entrypoint renders with the + # secret before nginx starts; this is the whole enforcement side. + # + # First in the block, before the trailing-slash rewrite below, so a refused + # request never gets as far as generating a redirect. Server level, not + # per-location: `if` at server level runs in the server-rewrite phase, i.e. + # BEFORE a location is chosen, so one line covers every location this block + # has and every location a later change adds. `return` is one of the two + # directives that are safe inside `if` (the `$is_bot` blocks below use the + # same shape). + # + # 421 rather than a direct `return 403`: an `error_page 403` would send the + # refusal back through the server-rewrite phase, hit this same `if` again + # and lose the custom page. A distinct internal code also keeps this trick + # and the crawler's 418 from ever being confused for one another. nginx + # generates a 421 of its own only for a coalesced HTTP/2 connection with a + # mismatched authority, which cannot happen here — Cloud Run speaks HTTP/1.1 + # to this container (the port is declared `http1`, not `h2c`). + error_page 421 = @origin_denied; + if ($origin_gate_deny) { + return 421; + } + # Use Google DNS for dynamic upstream resolution (prevents startup crash when domain not yet in DNS) resolver 8.8.8.8 8.8.4.4 valid=300s; resolver_timeout 5s; @@ -242,6 +266,25 @@ server { include /etc/nginx/security-headers.conf; } + # The refusal, as a page rather than nginx's stock one — which would + # otherwise announce the exact nginx version to anyone knocking on the raw + # origin. A named location is entered without re-running the server-rewrite + # phase, so it cannot loop back into the gate, and it is unreachable from + # outside by construction. `default_type` is what gives a `return` body its + # content type; the snippet is re-included because this location declares an + # add_header of its own and would otherwise drop every inherited one. + # + # It reports the verdict, never the value — `missing` and `mismatch` are the + # difference between "this path was never stamped" and "a rotation was only + # half applied", and telling a caller which of the two it is tells it + # nothing it did not already know about its own request. + location @origin_denied { + default_type text/html; + add_header X-Origin-Gate $origin_gate_status always; + include /etc/nginx/security-headers.conf; + return 403 '403 — not the front door

403

This is the origin of anyplot.ai, and it answers only requests that came through the edge.

'; + } + # Named location for bot SEO proxy location @seo_proxy { set $seo_backend https://api.anyplot.ai; @@ -326,11 +369,21 @@ server { try_files $uri $uri/ /index.html; } - # Health check endpoint for Cloud Run + # Health check endpoint for Cloud Run — and the origin gate's measuring + # instrument, the counterpart of the API's `GET /health`. It is the one path + # the gate exempts, so it answers on every route into this container + # whatever the gate is doing, and `X-Origin-Gate` reports the verdict for + # the request it was asked with (never the value). Ask it through the edge, + # through each hostname and on the raw `run.app` URL, and the rollout stops + # being a leap: arm only once every path that must keep working answers + # `off-seen`. `no-store` because a verdict cached at the edge is a lie about + # the next request. location /_health { access_log off; return 200 "OK"; add_header Content-Type text/plain; + add_header X-Origin-Gate $origin_gate_status always; + add_header Cache-Control "no-store" always; # This location's own add_header drops every inherited one, which is # the rule the top of security-headers.conf states and the one place in # this file that had missed it (found by tests/unit/api/test_csp_policy.py). @@ -360,6 +413,21 @@ server { # response so clients don't retry. if ($is_bot) { return 202; } if ($is_headless_client) { return 202; } + # The gate's second measuring point, and the only one that can see the + # apex Worker. `anyplot.ai/api/event` is the ONE path that Worker + # (infra/cloudflare/anyplot-api-proxy.js) sends to this container — + # everything else under /api/* goes to the API host — and a Worker + # subrequest to a host in the same zone skips that zone's Transform + # Rules, so there the header can only come from the Worker's own + # binding. Arming without having checked it would answer every Plausible + # pageview on the site with a 403, quietly. A mapped crawler UA gets the + # 202 above, so the probe costs no analytics event. + # + # Both server blocks carry it, though only the apex has that Worker + # route: a divergence between two copies of one location is how the copy + # nobody looks at rots. + add_header X-Origin-Gate $origin_gate_status always; + include /etc/nginx/security-headers.conf; proxy_pass https://plausible.io/api/event; proxy_set_header Host plausible.io; proxy_ssl_server_name on; @@ -390,6 +458,19 @@ server { listen 8080; server_name python.anyplot.ai; + # Origin gate, same shape and same reasoning as the main block above. + # Gated even though this hostname has neither a DNS record nor a Cloud Run + # domain mapping today (checked 2026-09-04), so nothing currently reaches + # this block at all: a gate that covers "every server block" is a rule + # someone can keep, while "every server block that is live" is a question + # nobody re-asks on the day the subdomain is switched on. Whoever switches + # it on adds the hostname to the Cloudflare Transform Rule in the same + # breath — the checklist is in infra/cloudflare/README.md. + error_page 421 = @origin_denied; + if ($origin_gate_deny) { + return 421; + } + resolver 8.8.8.8 8.8.4.4 valid=300s; resolver_timeout 5s; @@ -439,6 +520,16 @@ server { include /etc/nginx/security-headers.conf; } + # The refusal page, same as the main block — a named location belongs to one + # server block, so it has to exist in both or `error_page 421` here points + # at nothing. + location @origin_denied { + default_type text/html; + add_header X-Origin-Gate $origin_gate_status always; + include /etc/nginx/security-headers.conf; + return 403 '403 — not the front door

403

This is the origin of anyplot.ai, and it answers only requests that came through the edge.

'; + } + # Bot SEO proxy — proxy to backend. # /scatter-basic -> /seo-proxy/scatter-basic (hub, no language segment) # /scatter-basic/matplotlib -> /seo-proxy/scatter-basic/python/matplotlib (detail, language in path) @@ -502,10 +593,14 @@ server { try_files $uri $uri/ /index.html; } + # Same as the main block, gate verdict included: this hostname would be + # measured through its own edge route, so it needs its own instrument. location /_health { access_log off; return 200 "OK"; add_header Content-Type text/plain; + add_header X-Origin-Gate $origin_gate_status always; + add_header Cache-Control "no-store" always; # Same as the main block: an own add_header drops the inherited ones. include /etc/nginx/security-headers.conf; } @@ -531,6 +626,21 @@ server { # response so clients don't retry. if ($is_bot) { return 202; } if ($is_headless_client) { return 202; } + # The gate's second measuring point, and the only one that can see the + # apex Worker. `anyplot.ai/api/event` is the ONE path that Worker + # (infra/cloudflare/anyplot-api-proxy.js) sends to this container — + # everything else under /api/* goes to the API host — and a Worker + # subrequest to a host in the same zone skips that zone's Transform + # Rules, so there the header can only come from the Worker's own + # binding. Arming without having checked it would answer every Plausible + # pageview on the site with a 403, quietly. A mapped crawler UA gets the + # 202 above, so the probe costs no analytics event. + # + # Both server blocks carry it, though only the apex has that Worker + # route: a divergence between two copies of one location is how the copy + # nobody looks at rots. + add_header X-Origin-Gate $origin_gate_status always; + include /etc/nginx/security-headers.conf; proxy_pass https://plausible.io/api/event; proxy_set_header Host plausible.io; proxy_ssl_server_name on; diff --git a/app/origin-gate.conf.template b/app/origin-gate.conf.template new file mode 100644 index 00000000000..7c0cd01d216 --- /dev/null +++ b/app/origin-gate.conf.template @@ -0,0 +1,119 @@ +# The app container's half of the origin gate — the second door +# `api/origin_gate.py` named and could not close from its own side. +# +# `anyplot-app` stands with `ingress=all` and answers on two kinds of address: +# the hostnames Cloudflare proxies (anyplot.ai, www.anyplot.ai — both Cloud Run +# domain mappings) and the raw `*.run.app` URLs, which nothing proxies. On the +# raw URL there is no bot challenge, no WAF and no rate limit, and a crawler +# user agent is relayed through `@seo_proxy` to `https://api.anyplot.ai` — where +# the edge stamps the API's own secret legitimately, so the API gate sees a +# request that genuinely came through the front door. Every such relay costs a +# repository query on a cache miss and an outbound Plausible event. +# +# A Cloudflare Transform Rule stamps `X-Origin-Secret` on what it proxies for +# those hostnames; this file turns the header into a verdict, `nginx.conf` +# refuses on it. It is not authentication — it says "you came through the front +# door", nothing about who you are. +# +# WHY A HEADER AND NOT A HOST RULE. Both are possible here, which was not known +# when `api/origin_gate.py` wrote the question down: `anyplot.ai` and +# `www.anyplot.ai` are Cloud Run DOMAIN MAPPINGS, so Cloudflare forwards the +# original Host and `$host` really does distinguish the edge from the raw URL +# (Google's frontend answers a foreign Host on a `run.app` address with its own +# 404 before the container is reached, so the value cannot be spoofed). A host +# rule still cannot be the mechanism: `bot-serving-check.yml` probes this origin +# with crawler user agents precisely BECAUSE Cloudflare 403s GitHub-runner IPs, +# it cannot spoof the Host either, and any exception keyed on something public — +# a header it invents, a user agent — is public with this repository. The +# exception has to be the shared secret, and once the workflow carries the +# secret the host rule buys nothing the header does not. +# +# WHY THIS FILE IS A TEMPLATE. nginx cannot read the environment. The base image +# (`nginxinc/nginx-unprivileged:alpine`) ships the official entrypoint's +# `20-envsubst-on-templates.sh`, which renders `/etc/nginx/templates/*.template` +# into `/etc/nginx/conf.d/` before nginx starts — so the smallest mechanism is +# already in the image and needs no start script of our own. `app/Dockerfile` +# sets `NGINX_ENVSUBST_FILTER` so ONLY `ORIGIN_GATE` and `ORIGIN_SECRET` are +# substituted; without it envsubst would also eat `$host`, `$uri` and every +# other nginx variable whose name happens to exist in the environment. +# +# THE SECRET LANDS ON DISK HERE, and nowhere else in the container that is not +# already the environment. Two consequences worth knowing: `nginx -T` dumps +# every included file, so it must never be run where its output is published +# (CI runs `nginx -t`, which validates and prints nothing), and no `log_format`, +# `add_header` or `return` in `nginx.conf` may name `$http_x_origin_secret` — +# a test pins that. +# +# TWO DIFFERENCES FROM `api/secret_compare.py`, both deliberate and both +# harmless against a random 256-bit secret: nginx's `map` lowercases the source +# before hashing, so the comparison is case-INSENSITIVE, and a hash lookup is +# not constant-time. nginx offers no constant-time string primitive; the answer +# to both is the entropy of the secret, not the comparison. + +# What the caller presented, TAGGED. The tag is what makes an unset secret fail +# closed: `map` compares literally, so an empty `${ORIGIN_SECRET}` would produce +# the key `""` — which is exactly what an absent header looks like, and the gate +# would open for the whole internet the moment the variable was forgotten. With +# the tag, an empty secret expects the key `presented:`, and a header that is +# present but empty is reported as `absent` by the line below, so no request can +# produce it. +map $http_x_origin_secret $origin_gate_presented { + default "presented:$http_x_origin_secret"; + "" "absent"; +} + +map $origin_gate_presented $origin_gate_seen { + default 1; + "absent" 0; +} + +map $origin_gate_presented $origin_gate_match { + default 0; + "presented:${ORIGIN_SECRET}" 1; +} + +# The mode, carried in from the environment. `map` needs a variable to key on +# and `$host` is the cheapest one; the value is the same for every request. +# `app/Dockerfile` defaults it to `off`, so an unset variable on the service is +# the documented safe state and the rollback. +map $host $origin_gate_mode { + default "${ORIGIN_GATE}"; +} + +map $origin_gate_mode $origin_gate_armed { + default 0; + "on" 1; +} + +# Exempt by EXACT path, and only this one. `/_health` is the measurement +# endpoint below and the one path a probe must be able to reach on the raw URL +# whatever the gate is doing; a prefix exemption is how a gate quietly grows a +# hole, so `/_health/anything` is gated like everything else. Cloud Run's +# startup probe on this service is a `tcpSocket` check and needs no exemption +# at all (verified 2026-09-04) — this is for the deploy smoke and the daily bot +# monitor, both of which reach a `run.app` URL by definition. +map $uri $origin_gate_exempt { + default 0; + "/_health" 1; +} + +# The five verdicts of `api/origin_gate.py`, so both halves of one rollout are +# read the same way. `off-seen` is what makes arming measurable rather than +# brave: put the Transform Rule live while the gate is still off, ask every path +# in turn, and arm only once each one that must keep working reports the header +# arriving. `001` and `101` cannot occur — a match implies a header. +map "$origin_gate_armed$origin_gate_seen$origin_gate_match" $origin_gate_status { + default "off"; + "000" "off"; + "010" "off-seen"; + "011" "off-seen"; + "100" "missing"; + "110" "mismatch"; + "111" "ok"; +} + +map "$origin_gate_status:$origin_gate_exempt" $origin_gate_deny { + default 0; + "missing:0" 1; + "mismatch:0" 1; +} diff --git a/changelog.d/app-origin-gate.md b/changelog.d/app-origin-gate.md new file mode 100644 index 00000000000..283cbe986ff --- /dev/null +++ b/changelog.d/app-origin-gate.md @@ -0,0 +1,49 @@ +### Security + +- **The site's own origin has a gate now, and it ships switched off.** The API's + shared-secret gate closed one of two doors; `anyplot-app` stood with + `ingress=all` beside it, serving the whole site from its `*.run.app` URL with + no bot challenge, no WAF and no rate limit — and relaying any crawler user + agent through `@seo_proxy` into a repository query and a Plausible event. + `app/origin-gate.conf.template` is the nginx half: the base image's own + envsubst entrypoint renders the maps with the secret before nginx starts, and + every server block refuses what the Cloudflare edge did not stamp. + `ORIGIN_GATE` unset means off, `on` means 403, and armed with no secret fails + CLOSED — the map keys are tagged so an empty value cannot become "match + anything". Nothing is armed by merging: the rollout, the hostnames the + Transform Rule has to cover and the rollback are in + `infra/cloudflare/README.md`. + +- **`X-Origin-Gate` on `/_health` makes arming a measurement rather than a + leap.** The same five verdicts the API reports — `off`, `off-seen`, `ok`, + `missing`, `mismatch` — for the request they were asked with, never the value. + Ask every route into the container while the gate is still off and arm only + once each one that must keep working reports the header arriving. The apex + Worker's path cannot be asked that way, so `/api/event` reports it too: that + is the one path the Worker sends to this container, a Worker subrequest skips + its own zone's Transform Rules, and arming it blind would have answered every + Plausible pageview on the site with a 403. + +### Changed + +- **The three callers that reach the app origin without the edge now carry the + header themselves.** The pre-traffic smoke in `app/cloudbuild.yaml` reads + `ORIGIN_SECRET` inside the step rather than through `availableSecrets`, which + resolves at build start; `bot-serving-check.yml` sends it from the repository + secret and reads `/_health` first, so a missing or half-rotated secret fails + with a message naming itself instead of reporting ~36 crawler checks as a + broken site; and the apex Worker stamps its `/api/event` branch. A `Host` rule + would have been a real boundary here — `anyplot.ai` is a Cloud Run domain + mapping, so `$host` does tell the edge from the raw URL — but the bot monitor + cannot spoof a Host either, and any exception it could present instead is + public with this repository. + +### Added + +- **CI builds the app image and runs the gate against it.** `app/Dockerfile` was + hadolinted but never built before Cloud Build, which is after the merge — and + what it produces is not a program that fails to import but an nginx whose + config is now RENDERED at container start. The new job in `ci-image.yml` runs + the real image three ways: off, armed, and armed with no secret, checking the + 403, the exempt path, each verdict, and that the secret reaches neither the + refusal page nor the container log. diff --git a/docs/reference/api.md b/docs/reference/api.md index eace38ed1dc..8b3a053084c 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -627,16 +627,18 @@ to end while the gate is still off (below), not with an exemption: curl -s -A 'Mozilla/5.0 (compatible; Googlebot/2.1)' https://anyplot.ai/scatter-basic | head -5 ``` -**What the gate does not close.** It protects the API service's door. The app -service (`anyplot-app`) also stands with `ingress=all`, and its nginx relays a -crawler user agent through `@seo_proxy` to `https://api.anyplot.ai`, where the -edge stamps the header legitimately — so a caller who sends a crawler user -agent to the app's raw `*.run.app` URL still reaches the prerendered render and -its DB queries. That is a second door on a second service rather than a hole in -this one: the request the API sees really did come through the edge. Closing it -means gating the app service or refusing to proxy for `run.app` hosts in -`app/nginx.conf`, and `bot-serving-check.yml` probes that exact flow on the app -origin every night — so it is a separate change with its own blast radius. +**The second door, and where it is.** This gate protects the API service. The +app service (`anyplot-app`) also stands with `ingress=all`, and its nginx relays +a crawler user agent through `@seo_proxy` to `https://api.anyplot.ai`, where the +edge stamps the header legitimately — so a caller who sent a crawler user agent +to the app's raw `*.run.app` URL used to reach the prerendered render and its DB +queries, and this gate could not tell: the request it saw really did come +through the edge. That was never a hole in this one. It is a second door on a +second service, and it has a gate of its own now — +`app/origin-gate.conf.template`, the same secret and the same five verdicts, +reported on the app's `/_health` as `X-Origin-Gate`. Its rollout, the hostnames +it covers and the callers that reach that origin without the edge are in +[`infra/cloudflare/README.md`](../../infra/cloudflare/README.md#the-sites-own-origin-anyplot-app). **Observing it.** `GET /health` reports `origin_gate` for the request it was asked with, never the value: diff --git a/docs/reference/seo.md b/docs/reference/seo.md index 63427d5d16a..ed80eab57c9 100644 --- a/docs/reference/seo.md +++ b/docs/reference/seo.md @@ -184,6 +184,29 @@ location @seo_proxy { } ``` +### The crawler path only exists behind the edge + +Since the app's origin gate, every server block in `app/nginx.conf` refuses a +request that does not carry the `X-Origin-Secret` Cloudflare stamps — so this +whole path is reachable at `https://anyplot.ai/...` and nowhere else. That is +the point: on the raw `*.run.app` URL, the `$is_bot` map above was a free relay +into `@seo_proxy`, which is a repository query on a cache miss and an outbound +Plausible event per request, for anyone who set a user agent. + +Two consequences for anything that measures this path: + +- **`bot-serving-check.yml` sends the header** out of the `ORIGIN_SECRET` + repository secret, because it probes the Cloud Run origin on purpose + (Cloudflare 403s runner IPs even for a UA-spoofed Googlebot). It reads the + origin's `/_health` first and fails with a message naming the secret rather + than reporting ~36 403s as a broken crawler path. +- **A manual crawler probe goes through the edge**, which it always should have: + `curl -A "Googlebot" https://anyplot.ai/scatter-basic`. The same curl against + the `run.app` URL now answers 403 by design. + +The gate itself, its rollout and the hostnames it covers: +[`infra/cloudflare/README.md`](../../infra/cloudflare/README.md#the-sites-own-origin-anyplot-app). + ## SEO proxy endpoints Backend endpoints that serve HTML with correct meta tags for bots. diff --git a/infra/cloudflare/README.md b/infra/cloudflare/README.md index d5db974690f..bf8bd9efd12 100644 --- a/infra/cloudflare/README.md +++ b/infra/cloudflare/README.md @@ -1,10 +1,16 @@ -# Cloudflare: the apex Worker in front of the API +# Cloudflare: the edge in front of both Cloud Run services This directory is the source for Cloudflare configuration that would otherwise exist only in the dashboard. It was created with the origin gate (`api/origin_gate.py`), because until then the Worker went unmentioned in the repository and nobody without dashboard access could see what it does. +Two services stand behind the edge and both have an origin gate now. The API's +is in Python (`api/origin_gate.py`); the site's is in nginx +(`app/origin-gate.conf.template`). They share one secret, one set of five +verdicts, and one rollout procedure — the API's is done, the app's is +[at the end of this file](#the-sites-own-origin-anyplot-app). + ## What is here | File | Role | @@ -29,6 +35,14 @@ one path it does not forward is `/api/event`: that is the Plausible analytics endpoint (the same one `app/nginx.conf` proxies for the app service), and it goes to Plausible untouched. +**`/api/event` is the one path that reaches the SITE's origin**, and that makes +it the second place the same-zone finding below bites. `fetch(request)` on that +path goes to the app service's nginx, which has an origin gate of its own — and +being a Worker subrequest inside the zone, it carries no Transform Rule header. +So the Worker stamps that branch too. Without it, arming the app gate answers +every Plausible pageview on the site with a 403, quietly, because a page does +not tell its visitor that analytics failed. + **Why it stamps the origin secret itself.** This is the finding the directory exists for: @@ -186,3 +200,159 @@ curl -sI https://anyplot.ai/llms-full.txt After any change to the Worker, the Transform Rule or the secret: measure first, arm second. + +--- + +## The site's own origin (`anyplot-app`) + +The same door, on the other service. `anyplot-app` also stands with +`ingress=all`, so `https://anyplot-app-r3tvmejsmq-ez.a.run.app` serves the whole +site with no bot challenge, no WAF and no rate limit — and a crawler user agent +sent there is relayed by `@seo_proxy` to `https://api.anyplot.ai`, where the +edge stamps the API's secret legitimately. The API gate cannot see that: the +request it receives really did come through the front door. Every such relay is +a repository query on a cache miss plus an outbound Plausible event, on someone +else's terms. + +`app/origin-gate.conf.template` is the enforcing half. It is a template because +nginx cannot read the environment; the base image already ships the official +entrypoint's `20-envsubst-on-templates.sh`, so the secret arrives as an ordinary +Cloud Run environment variable and nothing new runs at container start. + +**Modes.** `ORIGIN_GATE` unset or anything but `on` = off; `ORIGIN_GATE=on` = +403 without a matching header. `ORIGIN_SECRET` is the value. Unset means off, +which is the rollback and the state the code ships in. `ORIGIN_GATE=on` with no +secret fails CLOSED — the map keys are tagged so an empty secret cannot become +"match anything". + +**Why a header and not a `Host` rule.** Both would work here, which was an open +question until 2026-09-04: `anyplot.ai` and `www.anyplot.ai` are Cloud Run +**domain mappings**, so Cloudflare forwards the original Host and `$host` really +does tell the edge from the raw URL — Google's frontend answers a foreign Host +on a `run.app` address with its own 404 before the container is reached, so the +value cannot be spoofed. A Host rule still cannot be the mechanism: +`bot-serving-check.yml` probes this origin with crawler user agents *because* +Cloudflare 403s GitHub-runner IPs, it cannot spoof the Host either, and any +exception keyed on something public — a header it invents, a user agent — is +public with this repository. The exception has to be the shared secret; and once +the workflow carries the secret, the Host rule buys nothing the header does not. + +### Hostnames this container serves + +The Transform Rule has to cover every one of them, or arming the gate locks out +the visitors it was meant to protect. + +| Hostname | Reaches the container via | Transform Rule | +|---|---|---| +| `anyplot.ai` | Cloud Run domain mapping, proxied by Cloudflare | **required** | +| `www.anyplot.ai` | Cloud Run domain mapping, proxied by Cloudflare — serves the site, it does not redirect | **required** | +| `anyplot.ai/api/event` | the apex Worker, `fetch(request)` to this origin | none: same-zone subrequest, the **Worker** stamps it | +| `python.anyplot.ai` | nothing today — `server_name` in `app/nginx.conf`, but no DNS record and no domain mapping (checked 2026-09-04) | add it in the same breath as the DNS record | +| `anyplot-app-r3tvmejsmq-ez.a.run.app` | direct | none — this is the door being closed | +| `anyplot-app-239660669828.europe-west4.run.app` | direct, the same service's second URL | none, same door | +| `candidate---anyplot-app-r3tvmejsmq-ez.a.run.app` | direct, the pre-traffic tag URL | none — `app/cloudbuild.yaml` sends the header itself | + +The rule is a **Set**, not an Add: a caller that supplies its own +`X-Origin-Secret` must have it replaced, not appended. + +### Callers that reach this origin without the edge + +Each one is legitimate, each one would be 403'd into silence, and each now +carries the header itself. + +| Caller | What it now sends | +|---|---| +| `app/cloudbuild.yaml` pre-traffic smoke | reads `ORIGIN_SECRET` from Secret Manager **inside the step** (not `availableSecrets`, which resolves at build start) and sends `X-Origin-Secret` on every probe. The build service account `239660669828-compute@developer.gserviceaccount.com` already holds `roles/secretmanager.secretAccessor` on the secret — it is the same account both triggers run as, and the API build already reads it. | +| `.github/workflows/bot-serving-check.yml` | sends the header from the `ORIGIN_SECRET` repository secret — the same one `sync-postgres.yml` already uses — and reads `/_health` first, so a missing or half-rotated secret fails with a message naming itself instead of reddening all ~36 crawler checks. | +| the apex Worker, `/api/event` | stamps from its own `ORIGIN_SECRET` binding (see above). | +| Cloud Run startup probe | nothing, and needs nothing: it is a `tcpSocket` check on 8080, not an HTTP probe (verified 2026-09-04). | +| IndexNow (`indexnow-submit.yml`, and Bing's verification fetch) | nothing, and needs nothing: both go to `https://anyplot.ai/.txt`, i.e. through the edge. | + +There are no Cloud Monitoring uptime checks on this project and no Lighthouse CI +(checked 2026-09-04); if one is added later it joins this table. + +### Measuring: `/_health` + +`/_health` is the gate's one exempt path — exact match, no prefix — and reports +`X-Origin-Gate` with the same five verdicts as the API's `/health`, for the +request it was asked with, never the value. + +```bash +curl -sI https://anyplot.ai/_health | grep -i x-origin-gate # edge, apex +curl -sI https://www.anyplot.ai/_health | grep -i x-origin-gate # edge, www +curl -sI https://anyplot-app-r3tvmejsmq-ez.a.run.app/_health | grep -i x-origin-gate +# the last one must stay "off" and become "missing": it is the closed door +``` + +The Worker's path cannot be asked through `/_health` — the Worker forwards only +`/api/event` to this origin — so it is measured on that endpoint instead. A +mapped crawler UA gets nginx's analytics shield (`202`) before Plausible is ever +called, so the probe costs no event: + +```bash +curl -si -X POST -A 'Mozilla/5.0 (compatible; Googlebot/2.1)' \ + https://anyplot.ai/api/event -d '{}' | grep -i -e '^HTTP' -e x-origin-gate +# 202 + "off-seen" the Worker is stamping — the state to reach before arming +# 202 + "off" the Worker is NOT stamping; arming now kills site analytics +``` + +### Rollout + +Ordered so that nothing is armed before it has been measured. Steps (a) and (b) +are safe on their own and can sit for days. + +```bash +# (a) merge and deploy. Nothing is armed: the image defaults ORIGIN_GATE=off, +# and the service declares no environment variables at all. +curl -sI https://anyplot.ai/_health | grep -i x-origin-gate # expect: off + +# (b) widen the Transform Rule to the app hostnames (dashboard: Rules → +# Transform Rules → Modify Request Header), and redeploy the Worker so its +# /api/event branch stamps too (see "Deploying" above). Then measure EVERY +# path — each one must read off-seen before anything is armed: +curl -sI https://anyplot.ai/_health | grep -i x-origin-gate +curl -sI https://www.anyplot.ai/_health | grep -i x-origin-gate +curl -si -X POST -A 'Googlebot' https://anyplot.ai/api/event -d '{}' | grep -i x-origin-gate +curl -sI https://anyplot-app-r3tvmejsmq-ez.a.run.app/_health | grep -i x-origin-gate # off + +# (c) the two callers. The repository secret already exists (sync-postgres.yml +# uses it) — confirm it, and confirm the build account can read the secret: +gh secret list --repo MarkusNeusinger/anyplot | grep ORIGIN_SECRET +gcloud secrets get-iam-policy ORIGIN_SECRET --project=anyplot +gh workflow run bot-serving-check.yml --repo MarkusNeusinger/anyplot # expect: "origin gate: off-seen" + +# (d) arm. A new revision is created and traffic moves to it; if the rendered +# config were invalid, nginx would not start, the revision would never +# become ready, and traffic would stay where it is. +gcloud run services update anyplot-app --region=europe-west4 --project=anyplot \ + --update-secrets=ORIGIN_SECRET=ORIGIN_SECRET:latest \ + --update-env-vars=ORIGIN_GATE=on + +# (e) verify, in this order: +curl -sI https://anyplot.ai/_health | grep -i x-origin-gate # ok +curl -s -o /dev/null -w '%{http_code}\n' https://anyplot.ai/ # 200 +curl -s -o /dev/null -w '%{http_code}\n' https://anyplot-app-r3tvmejsmq-ez.a.run.app/ # 403 +curl -s -A 'Mozilla/5.0 (compatible; Googlebot/2.1)' https://anyplot.ai/scatter-basic | grep canonical +curl -si -X POST -A 'Googlebot' https://anyplot.ai/api/event -d '{}' | head -1 # 202, not 403 +gh workflow run bot-serving-check.yml --repo MarkusNeusinger/anyplot # green + +# (f) rollback, either half on its own: +gcloud run services update anyplot-app --region=europe-west4 --project=anyplot \ + --remove-env-vars=ORIGIN_GATE +# or straight back to the revision that was serving before (d): +gcloud run revisions list --service anyplot-app --region europe-west4 --project anyplot \ + --format='table(name, creationTimestamp)' --limit 5 +gcloud run services update-traffic anyplot-app --region europe-west4 --project anyplot \ + --to-revisions==100 +``` + +`--update-secrets` and `--update-env-vars`, never the `--set-` forms: those +replace the whole set, so the next deploy would strip whatever was attached out +of band — the same trap `api/cloudbuild.yaml` documents for the API side. The +app's own `cloudbuild.yaml` names neither variable, so a deploy carries both +forward untouched. + +**Rotation** is roll back, rotate, arm again, and there are now **five** copies +of one value: Secret Manager, the API service, the app service, the Worker +binding and the GitHub repository secret. The gate is off in between, which is +the documented safe state. diff --git a/infra/cloudflare/anyplot-api-proxy.js b/infra/cloudflare/anyplot-api-proxy.js index 2c299e5699d..c5cb1b18417 100644 --- a/infra/cloudflare/anyplot-api-proxy.js +++ b/infra/cloudflare/anyplot-api-proxy.js @@ -4,8 +4,26 @@ export default { // The Plausible proxy shares this route. `/api/event` is the analytics // endpoint the site's nginx also proxies (app/nginx.conf); it is not an // anyplot API path and must reach Plausible untouched. + // + // It is also the ONE path under this route that goes to the SITE's origin + // instead of the API host — and the app has an origin gate of its own now + // (app/origin-gate.conf.template). The same finding applies a second time: + // a Worker subrequest to a host in the same zone skips that zone's + // Transform Rules, so the app gate can only ever see the header if it is + // stamped right here. Without these three lines, arming the app gate takes + // every Plausible pageview on anyplot.ai down — quietly, because analytics + // failing is not something a page tells its visitor about. + // + // Deleted before it is set, for the same reason as below: the headers are + // cloned from the incoming request, so a caller could otherwise supply its + // own value and have it forwarded whenever the binding is unset — which + // would make an unarmed probe report a false `off-seen` and corrupt the one + // measurement the rollout hangs on. if (url.pathname === '/api/event') { - return fetch(request); + const eventHeaders = new Headers(request.headers); + eventHeaders.delete('X-Origin-Secret'); + if (env.ORIGIN_SECRET) eventHeaders.set('X-Origin-Secret', env.ORIGIN_SECRET); + return fetch(new Request(request, { headers: eventHeaders })); } const targetPath = url.pathname.replace(/^\/api/, ''); const targetUrl = `https://api.anyplot.ai${targetPath}${url.search}`; diff --git a/tests/unit/api/test_app_origin_gate.py b/tests/unit/api/test_app_origin_gate.py new file mode 100644 index 00000000000..46ffd58e420 --- /dev/null +++ b/tests/unit/api/test_app_origin_gate.py @@ -0,0 +1,296 @@ +"""The APP container's origin gate, held against the six files it is spread over. + +`api/origin_gate.py` closes the API service's door and says, in as many words, +that the app service has a second one: its nginx relays a crawler user agent to +`https://api.anyplot.ai`, where the edge stamps the API's secret legitimately, +so a caller on the raw `*.run.app` URL reaches the prerendered render — and its +repository queries and its Plausible event — without having passed the edge +himself. This is that door, and `tests/unit/api/test_origin_gate.py` next door is +the other half of the same mechanism. + +The reason it needs a test at all is that no single file contains it: + +1. **`app/origin-gate.conf.template`** decides, and is the only place the secret + is written. It is a TEMPLATE because nginx cannot read the environment; the + base image's entrypoint renders it before nginx starts. +2. **`app/nginx.conf`** enforces, in every server block, and must never so much + as name the header — a `log_format`, an `add_header` or a `return` that did + would publish the secret the gate exists to keep. +3. **`app/Dockerfile`** is what makes the rendering happen, and carries the two + defaults without which the container does not boot at all. +4. **`app/cloudbuild.yaml`** and **`.github/workflows/bot-serving-check.yml`** + are the two legitimate callers that reach this origin WITHOUT the edge, by + design, and would each be 403'd into silence. +5. **`infra/cloudflare/anyplot-api-proxy.js`** forwards exactly one path to this + container, and a Worker subrequest skips its own zone's Transform Rules — so + it has to stamp that path itself or arming the gate takes site analytics down. + +None of the five can see the other four, and every one of them fails silently: +the site keeps serving, the deploy keeps going green, and what breaks is a +crawler path or an analytics call that nobody watches by eye. + +What this file cannot see is whether the rendered config actually boots — +`ci-image.yml` runs the real image with the gate off, armed, and armed with no +secret, which is the half a text test can never do. +""" + +from __future__ import annotations + +import re +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +NGINX_CONF = ROOT / "app" / "nginx.conf" +GATE_TEMPLATE = ROOT / "app" / "origin-gate.conf.template" +APP_DOCKERFILE = ROOT / "app" / "Dockerfile" +APP_CLOUDBUILD = ROOT / "app" / "cloudbuild.yaml" +BOT_CHECK = ROOT / ".github" / "workflows" / "bot-serving-check.yml" +WORKER = ROOT / "infra" / "cloudflare" / "anyplot-api-proxy.js" + + +def _without_comments(text: str) -> str: + """Comment lines dropped — prose that MENTIONS a directive is not one.""" + return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) + + +def server_blocks() -> list[str]: + """Every `server { … }` block of app/nginx.conf, as raw text.""" + conf = NGINX_CONF.read_text(encoding="utf-8") + blocks = [] + for match in re.finditer(r"^server\s*\{", conf, re.MULTILINE): + depth = 0 + for i in range(match.end() - 1, len(conf)): + if conf[i] == "{": + depth += 1 + elif conf[i] == "}": + depth -= 1 + if depth == 0: + blocks.append(conf[match.start() : i + 1]) + break + return blocks + + +def test_every_server_block_is_gated(): + """A gate on one of two server blocks is a door with a window beside it.""" + ungated = [ + b.strip().splitlines()[1].strip() for b in server_blocks() if "$origin_gate_deny" not in _without_comments(b) + ] + assert not ungated, ( + f"these app/nginx.conf server blocks refuse nothing: {ungated}. Every block " + "the container serves has to carry `if ($origin_gate_deny) { return 421; }`, " + "including one whose hostname is not live yet — a gate that covers 'every " + "server block' is a rule someone can keep." + ) + + +def test_the_refusal_has_somewhere_to_land_in_every_block(): + """`error_page` and the named location are per server block, not global.""" + for block in server_blocks(): + head = block.strip().splitlines()[1].strip() + text = _without_comments(block) + assert "error_page 421 = @origin_denied;" in text, ( + f"the server block at `{head}` refuses with 421 but never maps it — " + "the caller would get nginx's stock 421 page instead of the refusal." + ) + assert "location @origin_denied" in text, ( + f"the server block at `{head}` points `error_page 421` at a named " + "location it does not define. Named locations do not cross server blocks." + ) + + +def test_the_gate_refuses_before_the_trailing_slash_rewrite(): + """Rewrite-module directives run in source order. + + Behind the gate there is a `rewrite ^/(.*)/$ … permanent;` at server level. + Below the gate it never runs, which is the point: a refused request must not + first be handed a redirect to a URL it is equally not allowed to fetch. + """ + for block in server_blocks(): + text = _without_comments(block) + if "rewrite ^/(.*)/$" not in text: + continue + assert text.index("$origin_gate_deny") < text.index("rewrite ^/(.*)/$"), ( + "app/nginx.conf runs its trailing-slash rewrite before the origin gate, " + "so a refused request is answered with a redirect first." + ) + + +def test_nginx_conf_never_names_the_secret_header(): + """The one rule that keeps the secret out of a log line and an error page. + + `$http_x_origin_secret` belongs to the map file and nowhere else. The moment + it appears in a `log_format`, an `add_header` or a `return` body, the value + the whole gate protects is written into a place someone reads. + """ + conf = NGINX_CONF.read_text(encoding="utf-8") + assert "$http_x_origin_secret" not in conf, ( + "app/nginx.conf names $http_x_origin_secret. Only " + "app/origin-gate.conf.template may read the header; nginx.conf reads the " + "verdict ($origin_gate_status) and the decision ($origin_gate_deny)." + ) + + +def test_every_gate_variable_nginx_conf_uses_is_defined_by_the_template(): + """Two files, one mechanism: an undefined variable is a container that will + not start, and the container that will not start is the production one.""" + used = set(re.findall(r"\$origin_gate_\w+", NGINX_CONF.read_text(encoding="utf-8"))) + template = GATE_TEMPLATE.read_text(encoding="utf-8") + defined = set(re.findall(r"^map\s+\S+\s+(\$origin_gate_\w+)\s*\{", template, re.MULTILINE)) + assert used <= defined, ( + f"app/nginx.conf reads {sorted(used - defined)}, which " + "app/origin-gate.conf.template does not define — nginx refuses to start on " + "an unknown variable, so this is a boot failure, not a wrong answer." + ) + + +def test_the_secret_is_written_exactly_once_and_is_tagged(): + """An untagged key would open the gate the day the variable is forgotten. + + `map` compares literally. With a bare `"${ORIGIN_SECRET}"` key, an unset + variable renders as `""` — which is exactly what an absent header looks + like, so every request in the world would match and the gate would read as + armed while admitting everyone. The tag makes the empty-secret key + unreachable, because a present-but-empty header is reported as `absent`. + """ + # Comments dropped first: this file explains the trap at length, and prose + # about `${ORIGIN_SECRET}` must not count as another copy of it. + template = _without_comments(GATE_TEMPLATE.read_text(encoding="utf-8")) + occurrences = re.findall(r'"([^"]*)\$\{ORIGIN_SECRET\}([^"]*)"', template) + assert len(occurrences) == 1, ( + f"${{ORIGIN_SECRET}} appears in {len(occurrences)} map keys of " + "app/origin-gate.conf.template; it belongs in exactly one, and every extra " + "one is another copy of the secret in the rendered config." + ) + prefix, suffix = occurrences[0] + assert prefix and not suffix, ( + f'the secret\'s map key is "{prefix}${{ORIGIN_SECRET}}{suffix}" — it needs a ' + "non-empty PREFIX and nothing after it, or an unset secret becomes a key " + "that every request matches." + ) + assert template.count("${ORIGIN_SECRET}") == 1, ( + "the secret is substituted more than once in app/origin-gate.conf.template." + ) + + +def test_the_exemption_is_one_exact_path(): + """Exact paths only, no prefixes — a prefix exemption is how a gate grows a + hole, which is the rule `api/origin_gate.py` states for its own list.""" + template = _without_comments(GATE_TEMPLATE.read_text(encoding="utf-8")) + block = re.search(r"map \$uri \$origin_gate_exempt \{(.*?)\}", template, re.DOTALL) + assert block, "app/origin-gate.conf.template defines no exemption map" + exempt = re.findall(r'"([^"]+)"\s+1;', block.group(1)) + assert exempt == ["/_health"], ( + f"the origin gate exempts {exempt}. `/_health` is the measuring instrument " + "and the one path a probe must reach on the raw URL whatever the gate is " + "doing; anything else added here is a hole with a reason attached." + ) + + +def test_the_health_endpoint_reports_the_verdict_in_every_block(): + """`off-seen` is what makes arming a measurement rather than a leap, and a + verdict is only useful on the host it is asked about.""" + for block in server_blocks(): + head = block.strip().splitlines()[1].strip() + health = re.search(r"location /_health \{(.*?)\n \}", block, re.DOTALL) + assert health, f"the server block at `{head}` has no /_health location" + body = _without_comments(health.group(1)) + assert "add_header X-Origin-Gate $origin_gate_status always;" in body, ( + f"the /_health of `{head}` does not report the gate verdict, so that " + "route cannot be measured before the gate is armed." + ) + assert "include /etc/nginx/security-headers.conf;" in body, ( + f"the /_health of `{head}` sets a header of its own and so drops every " + "inherited one; it has to re-include the snippet." + ) + + +def test_the_worker_stamps_the_one_path_it_sends_to_this_container(): + """`/api/event` is the ONLY path under anyplot.ai/api/* that reaches the app. + + A Worker subrequest to a host in the same zone skips that zone's Transform + Rules — the finding `infra/cloudflare/README.md` exists for — so on this + path the header can only come from the Worker's own binding. Without it, + arming the app gate takes every Plausible pageview on the site down, and + analytics failing is not something a page tells its visitor about. + """ + worker = WORKER.read_text(encoding="utf-8") + branch = re.search(r"if \(url\.pathname === '/api/event'\) \{(.*?)\n \}", worker, re.DOTALL) + assert branch, "the Worker no longer has a distinct /api/event branch" + body = branch.group(1) + assert "env.ORIGIN_SECRET" in body and "set('X-Origin-Secret'" in body, ( + "the Worker's /api/event branch forwards without stamping X-Origin-Secret. " + "That path goes to the APP origin, and its gate will refuse it." + ) + assert "delete('X-Origin-Secret')" in body, ( + "the Worker's /api/event branch forwards the caller's own X-Origin-Secret. " + "It must be deleted before the binding is applied, or a caller can make an " + "unarmed probe report a false `off-seen` and corrupt the rollout's one " + "measurement." + ) + + +def test_the_deploy_smoke_carries_the_header(): + """The pre-traffic smoke probes the candidate on its `run.app` tag URL, which + by definition never passes the edge — so every probe but the exempt one has + to stamp the header itself, or arming the gate reds every deploy.""" + build = APP_CLOUDBUILD.read_text(encoding="utf-8") + assert "--secret=ORIGIN_SECRET" in build, ( + "app/cloudbuild.yaml never reads ORIGIN_SECRET, so its smoke probes the " + "candidate bare and would be 403'd the moment the gate is armed." + ) + bare = [line for line in build.splitlines() if re.search(r'curl -fsS \$\$RETRY (?!"\$\$\{HDR\[@\]\}")', line)] + assert not bare, f"these smoke probes do not send the origin header: {bare}" + + +def test_the_bot_monitor_carries_the_header_and_fails_loudly_without_it(): + """The daily monitor probes this origin BECAUSE Cloudflare 403s runner IPs. + + That is also why the gate could not be a Host rule. A missing repository + secret has to be an error that names itself — otherwise an armed gate turns + all ~36 checks into 403s and opens an incident saying every crawler page is + broken, which is the wrong thing to go looking for at 06:23 UTC. + """ + workflow = BOT_CHECK.read_text(encoding="utf-8") + assert "ORIGIN_SECRET: ${{ secrets.ORIGIN_SECRET }}" in workflow, ( + ".github/workflows/bot-serving-check.yml does not take the ORIGIN_SECRET " + "repository secret, so every probe it makes is refused once the gate is armed." + ) + assert 'HDR=(-H "X-Origin-Secret: ${ORIGIN_SECRET}")' in workflow, ( + "the monitor reads the secret but never sends it." + ) + for verdict in ("missing", "mismatch"): + assert re.search(rf"\n\s+{verdict}\)\n.*?::error::.*?\n\s+exit 1", workflow, re.DOTALL), ( + f"the monitor does not fail on an origin-gate verdict of `{verdict}` — " + "a missing or half-rotated secret would be reported as a broken site." + ) + bare = [ + line + for line in workflow.splitlines() + if re.search(r'curl -sS[^|]*"\$ORIGIN', line) and '"${HDR[@]}"' not in line + ] + assert not bare, f"these monitor probes do not send the origin header: {bare}" + + +def test_the_image_renders_the_template_and_defaults_both_variables(): + """Three Dockerfile lines the gate cannot work without, each failing + differently and none of them visibly. + + Without the COPY the maps never exist and nginx will not start. Without the + two ENV defaults envsubst leaves `${ORIGIN_SECRET}` standing as a literal — + nginx reads it as a reference to a variable it does not know, and again does + not start. Without the filter, envsubst rewrites every `$name` in the + template that happens to exist in the environment, and `$host` and `$uri` + there are nginx's variables, not the shell's. + """ + dockerfile = APP_DOCKERFILE.read_text(encoding="utf-8") + assert "/etc/nginx/templates/" in dockerfile and "origin-gate.conf.template" in dockerfile, ( + "app/Dockerfile does not put origin-gate.conf.template where the base " + "image's entrypoint looks for it (/etc/nginx/templates)." + ) + for setting in ("ORIGIN_GATE=off", 'ORIGIN_SECRET=""', "NGINX_ENVSUBST_FILTER="): + assert setting in dockerfile, ( + f"app/Dockerfile does not declare `{setting}`. All three are what make " + "an unset variable on the Cloud Run service mean 'gate off' instead of " + "'container will not boot'." + ) From 0151e9bc89c9f161562b0f3dc722e47a3f79abe6 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:21:11 +0200 Subject: [PATCH 2/4] chore(changelog): PR reference on the origin-gate fragment Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UEScQMZFvxxNNyNJYryfa3 --- changelog.d/app-origin-gate.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/changelog.d/app-origin-gate.md b/changelog.d/app-origin-gate.md index 283cbe986ff..7f74495aa5b 100644 --- a/changelog.d/app-origin-gate.md +++ b/changelog.d/app-origin-gate.md @@ -12,7 +12,7 @@ CLOSED — the map keys are tagged so an empty value cannot become "match anything". Nothing is armed by merging: the rollout, the hostnames the Transform Rule has to cover and the rollback are in - `infra/cloudflare/README.md`. + `infra/cloudflare/README.md`. (#11221) - **`X-Origin-Gate` on `/_health` makes arming a measurement rather than a leap.** The same five verdicts the API reports — `off`, `off-seen`, `ok`, @@ -22,7 +22,7 @@ Worker's path cannot be asked that way, so `/api/event` reports it too: that is the one path the Worker sends to this container, a Worker subrequest skips its own zone's Transform Rules, and arming it blind would have answered every - Plausible pageview on the site with a 403. + Plausible pageview on the site with a 403. (#11221) ### Changed @@ -36,7 +36,7 @@ would have been a real boundary here — `anyplot.ai` is a Cloud Run domain mapping, so `$host` does tell the edge from the raw URL — but the bot monitor cannot spoof a Host either, and any exception it could present instead is - public with this repository. + public with this repository. (#11221) ### Added @@ -46,4 +46,4 @@ config is now RENDERED at container start. The new job in `ci-image.yml` runs the real image three ways: off, armed, and armed with no secret, checking the 403, the exempt path, each verdict, and that the secret reaches neither the - refusal page nor the container log. + refusal page nor the container log. (#11221) From deada52890141de60a516ced7fd461d31164deaa Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:30:36 +0200 Subject: [PATCH 3/4] fix(gate): the armed map key needs a bigger hash bucket, and hadolint an exception MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings from the first CI run of the new container smoke, both of which only a running container could produce. nginx cannot hash a `map` key longer than one bucket, and the default bucket is the processor's cache line. The tagged key is `presented:` plus the whole secret, so a 32-byte secret in hex is 74 characters and nginx refuses to start with "could not build map_hash" — with the gate OFF it starts perfectly, because the key is short then, so the failure would have appeared at the exact moment of arming and nowhere earlier. The template now sets map_hash_bucket_size 512, the smoke uses a production-length secret so the ceiling stays exercised, and a test pins the directive. Hadolint DL3064 reads the ENV variable NAME and warns that a secret may be baked into the image. The value is the empty string, and it is declared for the opposite reason: so a service supplying no secret renders a config that refuses everyone rather than one nginx cannot parse. Renaming would silence the rule and break a four-place contract, so the exception sits on an ENV instruction of its own, with its reason beside it and the workflow's claim updated. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UEScQMZFvxxNNyNJYryfa3 --- .github/workflows/ci-image.yml | 20 +++++++++++++----- app/Dockerfile | 13 +++++++++++- app/origin-gate.conf.template | 16 +++++++++++++++ infra/cloudflare/README.md | 10 +++++++++ tests/unit/api/test_app_origin_gate.py | 28 ++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-image.yml b/.github/workflows/ci-image.yml index c83849b7c63..5d59c9188aa 100644 --- a/.github/workflows/ci-image.yml +++ b/.github/workflows/ci-image.yml @@ -235,9 +235,12 @@ jobs: # still caught — which a file-wide `ignore:` silently swallowed. The third # former exception, DL3013, is simply gone: `pip install uv==` is # pinned. DL3066 (non-numeric USER) still fires at info level, below the - # threshold, so it stays visible without blocking. app/Dockerfile needs no - # exceptions at all — verified against hadolint 2.15.1, the version this - # action pins. + # threshold, so it stays visible without blocking. app/Dockerfile declines + # exactly one, DL3064 on the origin gate's `ORIGIN_SECRET=""` default: the + # rule reads the variable NAME, the value is the empty string, and the + # reason it has to be declared at all is written beside the ignore. It sits + # on an ENV instruction of its own so it excuses that line and nothing + # else. Verified against hadolint 2.15.1, the version this action pins. - name: Hadolint (api/Dockerfile) if: steps.check.outputs.should_build == 'true' uses: hadolint/hadolint-action@06be81baf89a55ffd0e24b8f04a4185738dd3387 # v3.5.0 @@ -375,8 +378,15 @@ jobs: set -euo pipefail # Not a production value and never one: this container is thrown away # at the end of the job, and the real secret has no business on a - # runner that prints its own logs on failure. - SECRET="ci-origin-gate-not-a-production-secret" + # runner that prints its own logs on failure. It is the LENGTH of a + # real one on purpose — 64 characters, what `openssl rand -hex 32` + # produces. A short placeholder passed this job while the production + # length did not: the tagged map key is `presented:` plus the secret, + # and nginx cannot hash a key longer than one bucket, so with the gate + # armed the container refused to start. That is the bug this line now + # keeps caught (see map_hash_bucket_size in + # app/origin-gate.conf.template). + SECRET="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" start() { # start [docker run args...] local name="$1" port="$2"; shift 2 diff --git a/app/Dockerfile b/app/Dockerfile index 3a59df70974..d253990c4ec 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -55,9 +55,20 @@ COPY --chown=nginx:nginx origin-gate.conf.template /etc/nginx/templates/00-origi # shell's. The filter is a regex over variable NAMES; `^ORIGIN_` is exactly # ORIGIN_GATE and ORIGIN_SECRET. ENV ORIGIN_GATE=off \ - ORIGIN_SECRET="" \ NGINX_ENVSUBST_FILTER="^ORIGIN_" +# On its own instruction so the exception below excuses this line and nothing +# else. DL3064 reads the NAME and warns that a secret may be baked into the +# image; the value here is the empty string, and it is declared for the opposite +# reason — so that a service which supplies no secret renders a config that +# refuses everyone, rather than one nginx cannot parse. The real value arrives at +# runtime from Secret Manager and never reaches a layer. Renaming the variable +# would silence the rule and break the contract instead: the API service, the +# Worker binding, the GitHub repository secret and the Secret Manager entry all +# carry this name. +# hadolint ignore=DL3064 +ENV ORIGIN_SECRET="" + # Copy built assets from builder stage COPY --from=builder --chown=nginx:nginx /app/dist /usr/share/nginx/html diff --git a/app/origin-gate.conf.template b/app/origin-gate.conf.template index 7c0cd01d216..d30648453c8 100644 --- a/app/origin-gate.conf.template +++ b/app/origin-gate.conf.template @@ -50,6 +50,22 @@ # not constant-time. nginx offers no constant-time string primitive; the answer # to both is the entropy of the secret, not the comparison. +# nginx cannot hash a `map` key longer than one bucket, and the default bucket +# is the processor's cache line — 64 bytes on x86-64. The tagged key below is +# `presented:` plus the whole secret, so a 32-byte secret written as hex is 74 +# characters and nginx refuses to start with +# +# [emerg] could not build map_hash, you should increase map_hash_bucket_size +# +# — with the gate OFF it starts fine, because the key is short; with the gate +# armed the container never becomes ready. Cloud Run would keep the previous +# revision serving, so the failure is safe, but it is invisible until the moment +# of arming. Found by the container smoke in ci-image.yml, which is the only +# thing in this repository that could have found it. 512 leaves room for a +# base64 or longer secret, and is a multiple of the cache line as the directive +# asks; the cost is a slightly larger hash table for seven small maps. +map_hash_bucket_size 512; + # What the caller presented, TAGGED. The tag is what makes an unset secret fail # closed: `map` compares literally, so an empty `${ORIGIN_SECRET}` would produce # the key `""` — which is exactly what an absent header looks like, and the gate diff --git a/infra/cloudflare/README.md b/infra/cloudflare/README.md index bf8bd9efd12..c1bbf92f108 100644 --- a/infra/cloudflare/README.md +++ b/infra/cloudflare/README.md @@ -225,6 +225,16 @@ which is the rollback and the state the code ships in. `ORIGIN_GATE=on` with no secret fails CLOSED — the map keys are tagged so an empty secret cannot become "match anything". +**There is a length ceiling on the secret**, and it is worth knowing before a +rotation rather than during one. nginx cannot hash a `map` key longer than one +bucket; the key here is `presented:` plus the whole secret, and the template +therefore sets `map_hash_bucket_size 512`. A secret past roughly 500 characters +makes nginx refuse to start — with the gate *off* it starts fine, because the +key is short then, so the failure would appear only at the moment of arming. +Cloud Run keeps the previous revision serving in that case, so it is a safe +failure rather than an outage, and the container smoke in `ci-image.yml` runs a +production-length secret on every relevant change. + **Why a header and not a `Host` rule.** Both would work here, which was an open question until 2026-09-04: `anyplot.ai` and `www.anyplot.ai` are Cloud Run **domain mappings**, so Cloudflare forwards the original Host and `$host` really diff --git a/tests/unit/api/test_app_origin_gate.py b/tests/unit/api/test_app_origin_gate.py index 46ffd58e420..80953798943 100644 --- a/tests/unit/api/test_app_origin_gate.py +++ b/tests/unit/api/test_app_origin_gate.py @@ -144,6 +144,34 @@ def test_every_gate_variable_nginx_conf_uses_is_defined_by_the_template(): ) +def test_the_map_bucket_holds_a_real_secret(): + """The tag that makes the gate fail closed is also what overflows the hash. + + nginx cannot hash a `map` key longer than one bucket, and the default bucket + is the processor's cache line — 64 bytes. The key is `presented:` plus the + whole secret, so 32 random bytes written as hex is 74 characters and nginx + refuses to start with "could not build map_hash". It starts perfectly with + the gate off, because the key is short then; the failure appears at the + moment of arming and nowhere earlier. Found by the container smoke, which is + the only thing here that runs the rendered config. + """ + template = _without_comments(GATE_TEMPLATE.read_text(encoding="utf-8")) + match = re.search(r"map_hash_bucket_size\s+(\d+);", template) + assert match, ( + "app/origin-gate.conf.template does not raise map_hash_bucket_size. The " + "tagged secret key is longer than nginx's default 64-byte bucket, so the " + "container starts with the gate off and refuses to start the moment it is " + "armed." + ) + # `presented:` + a 64-character hex secret is 74; the headroom is for a + # longer or base64 value, and the directive wants a multiple of the cache + # line either way. + assert int(match.group(1)) >= 128, ( + f"map_hash_bucket_size is {match.group(1)}, which leaves no room for a " + "secret longer than a few dozen characters." + ) + + def test_the_secret_is_written_exactly_once_and_is_tagged(): """An untagged key would open the gate the day the variable is forgotten. From e34030dc731e38d908226817543552bc32d213a6 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:41:51 +0200 Subject: [PATCH 4/4] fix(gate): the secret stops at this server, and arming is a promote MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four review findings, three of them defects. The gate header was forwarded to every upstream. nginx passes incoming request headers to a proxied server by default, so once the edge stamps X-Origin-Secret on this host, the two Plausible locations in each server block would have handed the shared secret to a third party — and with it the API service's key, since both take the same value. Every proxy_pass in the file now clears it, which is one rule rather than a list of the dangerous ones, and a test refuses a location that proxies without it. Arming was written as two flags. This service pins traffic to a named revision, so `gcloud run services update` alone creates a revision that serves nothing while /_health still answers off. The runbook is now the API's own block: skip if a build is in flight, pin the SERVING image rather than the latest template, pin the secret to a NUMBER (with :latest a new version reaches new instances only, which reads as intermittent 403s inside one revision), then promote by name. Rolling back is its own block that looks nothing up, because the secret may be disabled during the incident it is undoing. A plain map key is matched without regard to case, so ORIGIN_GATE=ON arms as well. Kept, and documented instead of narrowed: nobody sets it to ON without meaning to arm, and the failure the other way — an operator who armed the gate, was told nothing and still has an open origin — is the one worth avoiding. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UEScQMZFvxxNNyNJYryfa3 --- app/nginx.conf | 43 +++++++++ app/origin-gate.conf.template | 6 ++ changelog.d/app-origin-gate.md | 6 +- infra/cloudflare/README.md | 125 +++++++++++++++++++++---- tests/unit/api/test_app_origin_gate.py | 42 +++++++++ 5 files changed, 201 insertions(+), 21 deletions(-) diff --git a/app/nginx.conf b/app/nginx.conf index 58b343de98a..82817052a75 100644 --- a/app/nginx.conf +++ b/app/nginx.conf @@ -290,6 +290,9 @@ server { set $seo_backend https://api.anyplot.ai; proxy_pass $seo_backend/seo-proxy$request_uri; proxy_set_header Host api.anyplot.ai; + # Consumed here, never forwarded — the rule every proxy_pass in this + # file follows; see location = /js/script.js for what it prevents. + proxy_set_header X-Origin-Secret ""; # Without this the upstream sees a plain-http request and any redirect # it generates is emitted as http://, downgrading the crawler's next # hop. The other proxied locations already set it. @@ -350,6 +353,8 @@ server { set $seo_backend https://api.anyplot.ai; proxy_pass $seo_backend/llms-full.txt; proxy_set_header Host api.anyplot.ai; + # Consumed here, never forwarded — see location = /js/script.js. + proxy_set_header X-Origin-Secret ""; proxy_set_header X-Forwarded-Proto https; proxy_ssl_server_name on; proxy_ssl_verify on; @@ -395,6 +400,12 @@ server { set $sitemap_backend https://api.anyplot.ai; proxy_pass $sitemap_backend/sitemap.xml; proxy_set_header Host api.anyplot.ai; + # Same rule as the Plausible hops: consumed here, never forwarded. This + # one goes to our own API, so the value would not have leaked — but the + # hop leaves this container over the public internet and re-enters + # through the edge, which stamps the header for that host itself. One + # rule for every proxy_pass is a rule that survives a new location. + proxy_set_header X-Origin-Secret ""; proxy_ssl_server_name on; } @@ -402,6 +413,15 @@ server { location = /js/script.js { proxy_pass https://plausible.io/js/pa-VsZKkJS49nA8_SUzShW2z.js; proxy_set_header Host plausible.io; + # THIRD PARTY. nginx forwards incoming request headers to an upstream by + # default, so once the edge stamps X-Origin-Secret on this host, every + # proxied script fetch would hand the shared secret to plausible.io — + # and with it the API's gate, since both services take the same value + # (Copilot review). An empty value means nginx sends no such header at + # all. The rule for this file is that the gate header is CONSUMED here + # and never forwarded, so every proxy_pass in both server blocks clears + # it and a test refuses one that does not. + proxy_set_header X-Origin-Secret ""; proxy_ssl_server_name on; proxy_buffering on; } @@ -430,6 +450,9 @@ server { include /etc/nginx/security-headers.conf; proxy_pass https://plausible.io/api/event; proxy_set_header Host plausible.io; + # Third party, same rule as /js/script.js above: the gate header is + # consumed by this server and never forwarded. + proxy_set_header X-Origin-Secret ""; proxy_ssl_server_name on; proxy_http_version 1.1; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -537,6 +560,8 @@ server { set $seo_backend https://api.anyplot.ai; proxy_pass $seo_backend$python_seo_uri; proxy_set_header Host api.anyplot.ai; + # Consumed here, never forwarded — see location = /js/script.js. + proxy_set_header X-Origin-Secret ""; proxy_ssl_server_name on; proxy_ssl_verify on; # Same 4-deep chain as @seo_proxy above — default depth 1 breaks it. @@ -609,12 +634,27 @@ server { set $sitemap_backend https://api.anyplot.ai; proxy_pass $sitemap_backend/sitemap.xml; proxy_set_header Host api.anyplot.ai; + # Same rule as the Plausible hops: consumed here, never forwarded. This + # one goes to our own API, so the value would not have leaked — but the + # hop leaves this container over the public internet and re-enters + # through the edge, which stamps the header for that host itself. One + # rule for every proxy_pass is a rule that survives a new location. + proxy_set_header X-Origin-Secret ""; proxy_ssl_server_name on; } location = /js/script.js { proxy_pass https://plausible.io/js/pa-VsZKkJS49nA8_SUzShW2z.js; proxy_set_header Host plausible.io; + # THIRD PARTY. nginx forwards incoming request headers to an upstream by + # default, so once the edge stamps X-Origin-Secret on this host, every + # proxied script fetch would hand the shared secret to plausible.io — + # and with it the API's gate, since both services take the same value + # (Copilot review). An empty value means nginx sends no such header at + # all. The rule for this file is that the gate header is CONSUMED here + # and never forwarded, so every proxy_pass in both server blocks clears + # it and a test refuses one that does not. + proxy_set_header X-Origin-Secret ""; proxy_ssl_server_name on; proxy_buffering on; } @@ -643,6 +683,9 @@ server { include /etc/nginx/security-headers.conf; proxy_pass https://plausible.io/api/event; proxy_set_header Host plausible.io; + # Third party, same rule as /js/script.js above: the gate header is + # consumed by this server and never forwarded. + proxy_set_header X-Origin-Secret ""; proxy_ssl_server_name on; proxy_http_version 1.1; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/app/origin-gate.conf.template b/app/origin-gate.conf.template index d30648453c8..94903afd96b 100644 --- a/app/origin-gate.conf.template +++ b/app/origin-gate.conf.template @@ -96,6 +96,12 @@ map $host $origin_gate_mode { default "${ORIGIN_GATE}"; } +# Case-INSENSITIVE, like every plain `map` key, so `On` and `ON` arm it too. +# That is the direction to be wrong in and it is chosen rather than inherited +# (Copilot review): nobody sets this variable to `ON` without meaning to arm, +# and the alternative failure — an operator who armed the gate, was told +# nothing, and is left with an origin that is still open — is the one worth +# avoiding. Anything that is not some casing of `on` leaves the gate off. map $origin_gate_mode $origin_gate_armed { default 0; "on" 1; diff --git a/changelog.d/app-origin-gate.md b/changelog.d/app-origin-gate.md index 7f74495aa5b..7de1f1684b7 100644 --- a/changelog.d/app-origin-gate.md +++ b/changelog.d/app-origin-gate.md @@ -10,8 +10,10 @@ every server block refuses what the Cloudflare edge did not stamp. `ORIGIN_GATE` unset means off, `on` means 403, and armed with no secret fails CLOSED — the map keys are tagged so an empty value cannot become "match - anything". Nothing is armed by merging: the rollout, the hostnames the - Transform Rule has to cover and the rollback are in + anything". The header is consumed by that server and never forwarded: every + `proxy_pass` clears it, so the Plausible hops cannot hand a third party the + key both services take. Nothing is armed by merging: the rollout, the + hostnames the Transform Rule has to cover and the rollback are in `infra/cloudflare/README.md`. (#11221) - **`X-Origin-Gate` on `/_health` makes arming a measurement rather than a diff --git a/infra/cloudflare/README.md b/infra/cloudflare/README.md index c1bbf92f108..8483456abb9 100644 --- a/infra/cloudflare/README.md +++ b/infra/cloudflare/README.md @@ -219,11 +219,13 @@ nginx cannot read the environment; the base image already ships the official entrypoint's `20-envsubst-on-templates.sh`, so the secret arrives as an ordinary Cloud Run environment variable and nothing new runs at container start. -**Modes.** `ORIGIN_GATE` unset or anything but `on` = off; `ORIGIN_GATE=on` = -403 without a matching header. `ORIGIN_SECRET` is the value. Unset means off, -which is the rollback and the state the code ships in. `ORIGIN_GATE=on` with no -secret fails CLOSED — the map keys are tagged so an empty secret cannot become -"match anything". +**Modes.** `ORIGIN_GATE` unset, or anything that is not some casing of `on`, = +off; `ORIGIN_GATE=on` (or `On`, or `ON` — a plain `map` key is matched without +regard to case, and that is the direction to be wrong in) = 403 without a +matching header. `ORIGIN_SECRET` is the value. Unset means off, which is the +rollback and the state the code ships in. `ORIGIN_GATE=on` with no secret fails +CLOSED — the map keys are tagged so an empty secret cannot become "match +anything". **There is a length ceiling on the secret**, and it is worth knowing before a rotation rather than during one. nginx cannot hash a `map` key longer than one @@ -331,12 +333,7 @@ gh secret list --repo MarkusNeusinger/anyplot | grep ORIGIN_SECRET gcloud secrets get-iam-policy ORIGIN_SECRET --project=anyplot gh workflow run bot-serving-check.yml --repo MarkusNeusinger/anyplot # expect: "origin gate: off-seen" -# (d) arm. A new revision is created and traffic moves to it; if the rendered -# config were invalid, nginx would not start, the revision would never -# become ready, and traffic would stay where it is. -gcloud run services update anyplot-app --region=europe-west4 --project=anyplot \ - --update-secrets=ORIGIN_SECRET=ORIGIN_SECRET:latest \ - --update-env-vars=ORIGIN_GATE=on +# (d) arm — the block below, not two flags. See "Arming, in full". # (e) verify, in this order: curl -sI https://anyplot.ai/_health | grep -i x-origin-gate # ok @@ -346,16 +343,106 @@ curl -s -A 'Mozilla/5.0 (compatible; Googlebot/2.1)' https://anyplot.ai/scatter- curl -si -X POST -A 'Googlebot' https://anyplot.ai/api/event -d '{}' | head -1 # 202, not 403 gh workflow run bot-serving-check.yml --repo MarkusNeusinger/anyplot # green -# (f) rollback, either half on its own: -gcloud run services update anyplot-app --region=europe-west4 --project=anyplot \ - --remove-env-vars=ORIGIN_GATE -# or straight back to the revision that was serving before (d): -gcloud run revisions list --service anyplot-app --region europe-west4 --project anyplot \ - --format='table(name, creationTimestamp)' --limit 5 -gcloud run services update-traffic anyplot-app --region europe-west4 --project anyplot \ - --to-revisions==100 +# (f) rollback — also its own block, below. ``` +### Arming, in full + +`gcloud run services update` alone is **not** the arm, and the reason is the +same one `docs/reference/api.md` § "Origin gate" writes out for the API: this +service pins traffic to a named revision (`app/cloudbuild.yaml` promotes with +`--to-revisions==100`), so an update creates a revision that serves +nothing, and `/_health` would still answer `off` while everything looked done +(Copilot review). Three more things each cost a comparable rollout somewhere, +so the block mirrors the API's, and runs fail-fast because half of these +commands feed the next one. + +```bash +( +set -euo pipefail +SERVICE=anyplot-app +LOC="--project=anyplot --region=europe-west4" + +# 0. Do not race the deploy pipeline: a build that already deployed its +# candidate promotes it at the end, and that revision was cloned from the +# pre-arm template — the promote would silently undo the arm, and its own +# smoke accepts `off` by design. +gcloud builds list --project=anyplot --region=europe-west4 --ongoing --format="value(id)" | grep -q . && { + echo "a Cloud Build is in flight; wait for it to finish (or fail) before arming." + exit 1 +} + +# 1. Build the new revision from the image that is SERVING, not from whatever +# is latest: `services update` clones the latest template, and this pipeline +# deliberately leaves each build's smoked-but-unpromoted candidate there. +read -r SERVING LATEST <<<"$(gcloud run services describe "$SERVICE" $LOC --format=json \ + | python3 -c "import json,sys; d=json.load(sys.stdin); \ + t=[x for x in d['status']['traffic'] if x.get('percent')==100]; \ + print(t[0]['revisionName'], d['status']['latestReadyRevisionName'])")" +IMAGE=$(gcloud run revisions describe "$SERVING" $LOC --format="value(spec.containers[0].image)") +test -n "$SERVING" && test -n "$IMAGE" || { echo "could not resolve the serving revision or its image"; exit 1; } +test "$SERVING" = "$LATEST" || echo "note: latest ($LATEST) is not serving ($SERVING) — image pinned to the serving one" + +# 2. Pin the secret to a NUMBER, never `:latest`. Cloud Run resolves a +# secret-backed variable when each instance starts, so with `:latest` a new +# secret version reaches new instances while older ones keep the old value — +# and since the edge stamps exactly one value, that shows up as intermittent +# 403s inside a single revision. +VERSION=$(gcloud secrets versions list ORIGIN_SECRET --project=anyplot \ + --filter="state=ENABLED" --sort-by=~createTime --limit=1 --format="value(name)") +test -n "$VERSION" || { echo "no ENABLED version of ORIGIN_SECRET"; exit 1; } + +# 3. Update, then promote BY NAME. `--to-latest` would hand traffic to whatever +# the pipeline last built. +SUFFIX="arm-$(date -u +%Y%m%d%H%M)" +gcloud run services update "$SERVICE" $LOC --image="$IMAGE" \ + --update-secrets="ORIGIN_SECRET=ORIGIN_SECRET:$VERSION" \ + --update-env-vars="ORIGIN_GATE=on" --revision-suffix="$SUFFIX" +gcloud run services update-traffic "$SERVICE" $LOC --to-revisions="$SERVICE-$SUFFIX=100" + +# 4. Confirm, and confirm which revision answered. A build that promoted over +# the arm shows up here as `off` on a path that carries the header. +curl -sI https://anyplot.ai/_health | grep -i x-origin-gate +gcloud run services describe "$SERVICE" $LOC --format="value(status.traffic)" +) +``` + +If the rendered config were invalid, nginx would not start, the revision would +never become ready, and the `update-traffic` would fail with traffic still on +the old revision — a safe failure, and the reason step 4 is not optional. + +### Rolling back + +Its own block, not the one above with a flag swapped: it has to run in the worst +state the service can be in, which includes the secret having been disabled +during the incident, so it looks nothing up. + +```bash +( +set -euo pipefail +SERVICE=anyplot-app +LOC="--project=anyplot --region=europe-west4" + +SERVING=$(gcloud run services describe "$SERVICE" $LOC --format=json \ + | python3 -c "import json,sys; d=json.load(sys.stdin); \ + print(next(x['revisionName'] for x in d['status']['traffic'] if x.get('percent')==100))") +IMAGE=$(gcloud run revisions describe "$SERVING" $LOC --format="value(spec.containers[0].image)") +test -n "$IMAGE" || { echo "could not resolve the serving image"; exit 1; } + +SUFFIX="disarm-$(date -u +%Y%m%d%H%M)" +gcloud run services update "$SERVICE" $LOC --image="$IMAGE" \ + --remove-env-vars=ORIGIN_GATE --revision-suffix="$SUFFIX" +gcloud run services update-traffic "$SERVICE" $LOC --to-revisions="$SERVICE-$SUFFIX=100" + +curl -sI https://anyplot.ai/_health | grep -i x-origin-gate # expect "off" or "off-seen" +) +``` + +Removing `ORIGIN_GATE` is enough; the secret may stay attached, which is what +makes re-arming one flag rather than two. Removing the Worker's binding is **not** +a rollback — while the service is armed, that takes `anyplot.ai/api/event` down +rather than freeing it. + `--update-secrets` and `--update-env-vars`, never the `--set-` forms: those replace the whole set, so the next deploy would strip whatever was attached out of band — the same trap `api/cloudbuild.yaml` documents for the API side. The diff --git a/tests/unit/api/test_app_origin_gate.py b/tests/unit/api/test_app_origin_gate.py index 80953798943..5bc6bd31067 100644 --- a/tests/unit/api/test_app_origin_gate.py +++ b/tests/unit/api/test_app_origin_gate.py @@ -144,6 +144,48 @@ def test_every_gate_variable_nginx_conf_uses_is_defined_by_the_template(): ) +def location_blocks() -> list[str]: + """Every `location …{ … }` block of app/nginx.conf, as raw text.""" + conf = NGINX_CONF.read_text(encoding="utf-8") + blocks = [] + for match in re.finditer(r"^\s*location\s[^{]*\{", conf, re.MULTILINE): + depth = 0 + for i in range(match.end() - 1, len(conf)): + if conf[i] == "{": + depth += 1 + elif conf[i] == "}": + depth -= 1 + if depth == 0: + blocks.append(conf[match.start() : i + 1]) + break + return blocks + + +def test_no_upstream_is_ever_handed_the_gate_header(): + """nginx forwards incoming request headers to an upstream by default. + + So the moment the edge stamps `X-Origin-Secret` on this host, every location + that proxies would pass the shared secret on — to `plausible.io` among + others, a third party that would thereby also hold the API service's key + (Copilot review). The rule is that the header is CONSUMED by this server and + never forwarded, which is one rule for every proxy_pass rather than a list of + the dangerous ones; `proxy_set_header X-Origin-Secret ""` makes nginx send no + such header at all. + """ + leaking = [] + for block in location_blocks(): + text = _without_comments(block) + if not re.search(r"^\s*proxy_pass\s", text, re.MULTILINE): + continue + if 'proxy_set_header X-Origin-Secret "";' not in text: + leaking.append(block.strip().splitlines()[0].strip()) + assert not leaking, ( + f"these app/nginx.conf locations proxy without clearing the gate header: " + f'{leaking}. Add `proxy_set_header X-Origin-Secret "";` — nginx forwards ' + "request headers by default, so the upstream would receive the secret." + ) + + def test_the_map_bucket_holds_a_real_secret(): """The tag that makes the gate fail closed is also what overflows the hash.