From 50f5bb69c040594fd93fb1281bce6ea7a2ddc509 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:09:33 -0700 Subject: [PATCH 01/25] feat(schedule): record run state in both runner modes and stop treating a skip as failure Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule.go | 62 ++++++++++++++++++++++++-------- internal/engine/schedule_test.go | 40 +++++++++++++++++++++ 2 files changed, 87 insertions(+), 15 deletions(-) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 2be48e8..60b01b9 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -178,7 +178,7 @@ func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Na compose := "/usr/bin/docker compose -p " + q(application) + " --project-directory " + projectDir + " -f " + projectDir + "/" + q("compose.yaml") + scheduleRuntimeEnvArgs(projectDir, runtimeEnvFiles) + " run --rm --no-deps --name " + q(container) + " " + q(job.Name) - return strings.Join([]string{ + lines := []string{ "#!/bin/sh", "# Written by Onebox. Edits are overwritten on the next deploy.", "set -eu", @@ -188,15 +188,20 @@ func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Na "exec 8>" + q(names.ScheduleRunLock()), "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 8", "if [ -e " + q(applicationLock) + " ]; then echo 'onebox: an application operation holds the deploy lock' >&2; exit 75; fi", + // Best effort: the record names the release that ran, and an exclusive + // job runs whatever `current` points at when it starts. + "release_dir=$(readlink -f " + q(names.CurrentLink()) + " 2>/dev/null || true)", + "release=${release_dir##*/}", scheduleContainerCleanup(container), - "cleanup() { " + scheduleContainerCleanup(container) + "; }", + "cleanup() { " + scheduleContainerCleanup(container) + "; rm -f \"$tmp\"; }", "trap cleanup 0", "trap 'exit 129' 1", "trap 'exit 130' 2", "trap 'exit 143' 15", - compose, - "", - }, "\n") + } + lines = append(lines, scheduleRunPreamble(names.ScheduledJobRunState(job.Name))...) + lines = append(lines, "write_state 1", compose, "") + return strings.Join(lines, "\n") } func pinnedScheduleRunnerScript(application, job string, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile) string { @@ -226,21 +231,14 @@ func pinnedScheduleRunnerScript(application, job string, names app.Names, applic "chmod 600 \"$release_dir/.ob-schedule.lease\"", "/usr/bin/flock --shared 7", scheduleContainerCleanup(container), - "state=" + q(state), - "tmp=\"$state.$$\"", - "cleanup() { " + scheduleContainerCleanup(container) + "; rm -f \"$state\" \"$tmp\"; }", + "cleanup() { " + scheduleContainerCleanup(container) + "; rm -f \"$tmp\"; }", "trap cleanup 0", "trap 'exit 129' 1", "trap 'exit 130' 2", "trap 'exit 143' 15", - "started_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", - "umask 077", - "printf 'release=%s\\nstarted_at=%s\\n' \"$release\" \"$started_at\" >\"$tmp\"", - "mv -f \"$tmp\" \"$state\"", - "/usr/bin/flock --unlock 8", - compose, - "", } + lines = append(lines, scheduleRunPreamble(state)...) + lines = append(lines, "write_state 1", "/usr/bin/flock --unlock 8", compose, "") return strings.Join(lines, "\n") } @@ -248,6 +246,36 @@ func scheduleContainerCleanup(container string) string { return "/usr/bin/docker rm -f " + q(container) + " >/dev/null 2>&1 || true" } +// scheduleStateFunction renders the shell function both runners use to record +// the run in progress. The notifier reads it after the run ends, so the runner +// never removes it: a runner that cleaned up its own state would erase the +// only evidence a timed-out run leaves behind. +func scheduleStateFunction() []string { + return []string{ + "write_state() {", + " umask 077", + " printf 'release=%s\\nstarted_at=%s\\nstarted_epoch=%s\\ntrigger=%s\\noperation=%s\\nattempt=%s\\ninputs=%s\\n' " + + "\"$release\" \"$started_at\" \"$started_epoch\" \"$trigger\" \"$operation\" \"$1\" \"$inputs_json\" >\"$tmp\"", + " mv -f \"$tmp\" \"$state\"", + "}", + } +} + +// scheduleRunPreamble sets the variables write_state records. The trigger is +// systemd's own word for it: a timer activation carries TRIGGER_UNIT (systemd +// 252 and newer), anything else is an operator. +func scheduleRunPreamble(state string) []string { + return append([]string{ + "state=" + q(state), + "tmp=\"$state.$$\"", + "started_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", + "started_epoch=$(date -u '+%s')", + "if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=manual; fi", + "operation=''", + "inputs_json=''", + }, scheduleStateFunction()...) +} + func scheduleRuntimeEnvArgs(projectDir string, entries []app.EnvFile) string { args := "" for _, entry := range entries { @@ -275,6 +303,10 @@ func scheduleServiceUnit(application string, job app.ScheduledJob, runnerPath, n // whether failure notifications are needed. "ExecStopPost=/bin/sh " + notifyPath, "TimeoutStartSec=" + job.Timeout, + // Exit 75 is the runner's "skipped for a lock conflict". It is a fact + // about timing, not a failure of the job, and it must not leave the + // unit failed or trip failure notifications. + "SuccessExitStatus=75", "", }, "\n") } diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index 13b35fa..ac083d8 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -822,3 +822,43 @@ func TestAppNamedBackupDoesNotOwnEveryBackupTimer(t *testing.T) { t.Fatalf("app named backup removed another application's timer:\n%s", seq) } } + +func TestScheduledJobRunnersRecordRunStateForTheNotifier(t *testing.T) { + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + for _, tc := range []struct { + name string + job app.ScheduledJob + }{ + {"exclusive", app.ScheduledJob{Name: "nightly", Cron: "0 2 * * *", Timezone: "UTC", Calendar: "*-*-* 02:00:00", Timeout: "45m", DeployLock: "exclusive"}}, + {"pinned", app.ScheduledJob{Name: "nightly", Cron: "0 2 * * *", Timezone: "UTC", Calendar: "*-*-* 02:00:00", Timeout: "45m", DeployLock: "pinned"}}, + } { + t.Run(tc.name, func(t *testing.T) { + runner := scheduleRunnerScript("sample", tc.job, names, "/var/lib/ob/sample/lock", nil) + for _, want := range []string{ + "state='/var/lib/ob/sample/schedule/nightly.state'", + "write_state() {", + "started_epoch=%s", + "trigger=%s", + "write_state 1", + "mv -f \"$tmp\" \"$state\"", + } { + if !strings.Contains(runner, want) { + t.Errorf("%s runner is missing %q:\n%s", tc.name, want, runner) + } + } + if strings.Contains(runner, `rm -f "$state"`) { + t.Errorf("%s runner removes the state the notifier finalises:\n%s", tc.name, runner) + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(runner) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("%s runner is not valid POSIX shell: %v: %s\n%s", tc.name, err, output, runner) + } + }) + } + service := scheduleServiceUnit("sample", app.ScheduledJob{Name: "nightly", Timeout: "45m"}, + "/etc/systemd/system/ob-sample-nightly.run", "/etc/systemd/system/ob-sample-nightly.notify") + if !strings.Contains(service, "SuccessExitStatus=75") { + t.Errorf("a lock-conflict skip must not be a failed unit:\n%s", service) + } +} From f138dfa110ab1f6e4630fb6aaaa7f42ca4ab2661 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:11:52 -0700 Subject: [PATCH 02/25] feat(schedule): finalise one journal record per scheduled run from ExecStopPost Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule.go | 54 ++++++++++++- internal/engine/schedule_test.go | 129 +++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 1 deletion(-) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 60b01b9..55b0598 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -313,6 +313,57 @@ func scheduleServiceUnit(application string, job app.ScheduledJob, runnerPath, n const scheduleNotificationTimestamp = "__ONEBOX_SCHEDULE_TIMESTAMP__" +// scheduleRunIdentifier is the syslog identifier of the one line the notifier +// writes per run. `journalctl -u -t ob-run` is the run history: the +// journal is the store, so there is no file to trim and nothing that can +// disagree with the unit's own log. +const scheduleRunIdentifier = "ob-run" + +// scheduleRunRecordLines finalises the run the runner started. This lives in +// ExecStopPost because only systemd knows how the run ended: a timed-out +// runner is killed mid-sleep and cannot write its own outcome. Every value +// interpolated into the JSON is either numeric, a timestamp the runner +// formatted, a release id, or an input value the loader restricted to a +// charset that needs no escaping. +func scheduleRunRecordLines(job, state string) []string { + return []string{ + "state=" + q(state), + "release=''; started_at=''; started_epoch=''; trigger=''; operation=''; attempt=0; inputs=''", + "if [ -f \"$state\" ]; then", + " while IFS= read -r line || [ -n \"$line\" ]; do", + " case \"$line\" in", + " release=*) release=${line#release=} ;;", + " started_at=*) started_at=${line#started_at=} ;;", + " started_epoch=*) started_epoch=${line#started_epoch=} ;;", + " trigger=*) trigger=${line#trigger=} ;;", + " operation=*) operation=${line#operation=} ;;", + " attempt=*) attempt=${line#attempt=} ;;", + " inputs=*) inputs=${line#inputs=} ;;", + " esac", + " done <\"$state\"", + " rm -f \"$state\"", + "fi", + "if [ -z \"$trigger\" ]; then if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=manual; fi; fi", + "result=${SERVICE_RESULT:-success}", + "status=${EXIT_STATUS:-0}", + // EXIT_STATUS is a signal name when the main process was killed. + "case \"$status\" in ''|*[!0-9]*) status=null ;; esac", + "case \"$attempt\" in ''|*[!0-9]*) attempt=0 ;; esac", + "if [ \"$result\" = timeout ]; then outcome=timeout", + "elif [ \"$status\" = 75 ]; then outcome=skipped", + "elif [ \"$result\" = success ] && [ \"$status\" = 0 ]; then outcome=success", + "else outcome=failure; fi", + "finished_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", + "now=$(date -u '+%s')", + "duration=0", + "case \"$started_epoch\" in ''|*[!0-9]*) ;; *) duration=$((now - started_epoch)) ;; esac", + "[ -z \"$started_at\" ] && started_at=$finished_at", + "printf '{\"run\":\"%s\",\"job\":\"%s\",\"trigger\":\"%s\",\"operation\":\"%s\",\"release\":\"%s\",\"started_at\":\"%s\",\"finished_at\":\"%s\",\"duration_s\":%s,\"attempts\":%s,\"exit_status\":%s,\"outcome\":\"%s\",\"inputs\":{%s}}\\n' " + + "\"${INVOCATION_ID:-}\" " + q(job) + " \"$trigger\" \"$operation\" \"$release\" \"$started_at\" \"$finished_at\" \"$duration\" \"$attempt\" \"$status\" \"$outcome\" \"$inputs\" " + + "| systemd-cat -t " + scheduleRunIdentifier + " || true", + } +} + // scheduleFailureNotifier extends the existing notification contract to work // fired directly by systemd. The generated file is mode 0600, keeping webhook // tokens out of unit metadata, and every send is bounded and fail-open. @@ -329,8 +380,9 @@ func (e *Engine) scheduleFailureNotifier(job string) (string, error) { "if /usr/bin/flock --exclusive --nonblock 9; then", " " + scheduleContainerCleanup(e.names().Container(job, 1)), "fi", - `[ "${SERVICE_RESULT:-success}" = success ] && exit 0`, } + lines = append(lines, scheduleRunRecordLines(job, e.names().ScheduledJobRunState(job))...) + lines = append(lines, `case "$outcome" in failure|timeout) ;; *) exit 0 ;; esac`) var sends []string for _, name := range sortedNames(e.Spec.Notifications) { cfg := e.Spec.Notifications[name] diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index ac083d8..6b30283 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -3,6 +3,7 @@ package engine import ( "bytes" "context" + "encoding/json" "os" "os/exec" "path/filepath" @@ -862,3 +863,131 @@ func TestScheduledJobRunnersRecordRunStateForTheNotifier(t *testing.T) { t.Errorf("a lock-conflict skip must not be a failed unit:\n%s", service) } } + +func TestScheduledJobNotifierWritesOneRunRecordToTheJournal(t *testing.T) { + cfg := testConfig() + f := &transport.Fake{TargetName: "root@example.internal"} + e := New(cfg, testProject(t), f, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + script, err := e.scheduleFailureNotifier("nightly") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "state='/var/lib/ob/sample/schedule/nightly.state'", + `started_epoch=*) started_epoch=${line#started_epoch=}`, + `rm -f "$state"`, + `result=${SERVICE_RESULT:-success}`, + `status=${EXIT_STATUS:-0}`, + `outcome=timeout`, + `outcome=skipped`, + `outcome=success`, + `outcome=failure`, + `"run":"%s","job":"%s","trigger":"%s","operation":"%s","release":"%s"`, + `"duration_s":%s,"attempts":%s,"exit_status":%s,"outcome":"%s","inputs":{%s}`, + `"${INVOCATION_ID:-}" 'nightly'`, + "systemd-cat -t ob-run", + } { + if !strings.Contains(script, want) { + t.Errorf("notifier is missing %q:\n%s", want, script) + } + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(script) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("notifier is not valid POSIX shell: %v: %s\n%s", err, output, script) + } +} + +// runNotifier executes the generated ExecStopPost script with a stub +// systemd-cat, the way systemd would after a run. It returns the record the +// script wrote and whether the state file survived. +func runNotifier(t *testing.T, state string, env map[string]string) (map[string]any, bool) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("POSIX shell required") + } + base := t.TempDir() + cfg := testConfig() + cfg.BasePath = base + cfg.Notifications = nil + e := New(cfg, testProject(t), &transport.Fake{TargetName: "root@example.internal"}, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + script, err := e.scheduleFailureNotifier("nightly") + if err != nil { + t.Fatal(err) + } + scheduleDir := filepath.Join(base, "sample", "schedule") + if err := os.MkdirAll(scheduleDir, 0o700); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(scheduleDir, "nightly.state") + if state != "" { + if err := os.WriteFile(statePath, []byte(state), 0o600); err != nil { + t.Fatal(err) + } + } + bin := t.TempDir() + record := filepath.Join(bin, "record.jsonl") + stub := "#!/bin/sh\n[ \"$1\" = -t ] && [ \"$2\" = ob-run ] || exit 9\ncat >>" + record + "\n" + if err := os.WriteFile(filepath.Join(bin, "systemd-cat"), []byte(stub), 0o755); err != nil { + t.Fatal(err) + } + command := exec.CommandContext(context.Background(), "sh", "-s") + command.Stdin = strings.NewReader(script) + command.Env = append([]string{"PATH=" + bin + ":" + os.Getenv("PATH")}, "HOME="+base) + for k, v := range env { + command.Env = append(command.Env, k+"="+v) + } + if out, err := command.CombinedOutput(); err != nil { + t.Fatalf("notifier exited non-zero: %v\n%s\n%s", err, out, script) + } + body, err := os.ReadFile(record) + if err != nil { + t.Fatalf("notifier wrote no record: %v", err) + } + lines := strings.Split(strings.TrimSpace(string(body)), "\n") + if len(lines) != 1 { + t.Fatalf("notifier wrote %d records, want 1:\n%s", len(lines), body) + } + var decoded map[string]any + if err := json.Unmarshal([]byte(lines[0]), &decoded); err != nil { + t.Fatalf("record is not JSON: %v\n%s", err, lines[0]) + } + _, stateErr := os.Stat(statePath) + return decoded, stateErr == nil +} + +func TestScheduledJobNotifierRecordsEachOutcomeAndRemovesState(t *testing.T) { + state := "release=20260905-140000-ab12cd\nstarted_at=2026-09-05T15:00:01Z\nstarted_epoch=1\ntrigger=timer\noperation=\nattempt=2\ninputs=\n" + for name, tc := range map[string]struct { + state string + env map[string]string + outcome string + exit any + attempt float64 + }{ + "success": {state, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "a1b2"}, "success", float64(0), 2}, + "failure": {state, map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "1"}, "failure", float64(1), 2}, + "timeout": {state, map[string]string{"SERVICE_RESULT": "timeout", "EXIT_STATUS": "TERM"}, "timeout", nil, 2}, + "skipped": {"", map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "75", "TRIGGER_UNIT": "ob-sample-nightly.timer"}, "skipped", float64(75), 0}, + "no state": {"", map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "3"}, "failure", float64(3), 0}, + } { + t.Run(name, func(t *testing.T) { + record, stateLeft := runNotifier(t, tc.state, tc.env) + if record["outcome"] != tc.outcome || record["exit_status"] != tc.exit || record["attempts"] != tc.attempt { + t.Fatalf("record = %#v", record) + } + if record["job"] != "nightly" || record["run"] != tc.env["INVOCATION_ID"] { + t.Fatalf("identity fields wrong: %#v", record) + } + if tc.state != "" && (record["release"] != "20260905-140000-ab12cd" || record["trigger"] != "timer" || record["duration_s"].(float64) < 1) { + t.Fatalf("state fields not carried: %#v", record) + } + if name == "skipped" && record["trigger"] != "timer" { + t.Fatalf("trigger not derived from TRIGGER_UNIT: %#v", record) + } + if stateLeft { + t.Fatal("state file survived the notifier") + } + }) + } +} From 62af20661b3f981478383b5c86758d16b90e0bcf Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:12:59 -0700 Subject: [PATCH 03/25] feat(schedule): read run records, timer state and per-run logs from the host journal Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule_history.go | 184 +++++++++++++++++++++++ internal/engine/schedule_history_test.go | 106 +++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 internal/engine/schedule_history.go create mode 100644 internal/engine/schedule_history_test.go diff --git a/internal/engine/schedule_history.go b/internal/engine/schedule_history.go new file mode 100644 index 0000000..2a82c89 --- /dev/null +++ b/internal/engine/schedule_history.go @@ -0,0 +1,184 @@ +package engine + +import ( + "context" + "encoding/json" + "fmt" + "io" + "regexp" + "strconv" + "strings" + + "github.com/labstack/onebox/internal/app" +) + +// ScheduleRunRecord is the line the notifier writes to the journal when a +// scheduled run ends. The journal is the store: there is no file to trim and +// nothing that can disagree with the unit's own log. +type ScheduleRunRecord struct { + Run string `json:"run"` + Job string `json:"job"` + Trigger string `json:"trigger"` + Operation string `json:"operation,omitempty"` + Release string `json:"release,omitempty"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + DurationSeconds int `json:"duration_s"` + Attempts int `json:"attempts"` + ExitStatus *int `json:"exit_status"` + Outcome string `json:"outcome"` + Inputs map[string]string `json:"inputs,omitempty"` +} + +// ScheduleListing is one declared job beside its timer as the host reports it. +type ScheduleListing struct { + Name string `json:"name"` + Unit string `json:"unit"` + Cron string `json:"cron"` + Timezone string `json:"timezone"` + DeployLock string `json:"deploy_lock"` + Timeout string `json:"timeout"` + TimerState string `json:"timer_state"` + NextRun string `json:"next_run,omitempty"` + LastTrigger string `json:"last_trigger,omitempty"` +} + +// A run id is systemd's invocation id. It reaches a shell as a journalctl +// match, so it is checked against the only shape systemd produces. +var scheduleRunID = regexp.MustCompile(`^[0-9a-f]{32}$`) + +func scheduleHistoryCommand(unit string, n int) string { + if n <= 0 { + n = 20 + } + return "journalctl -u " + q(unit+".service") + " -t " + scheduleRunIdentifier + + " -o cat -r -n " + strconv.Itoa(n) + " --no-pager 2>/dev/null || true" +} + +// parseScheduleRunRecords keeps the lines that decode and drops the rest. A +// host with a hand-edited unit or an older notifier may leave other text under +// the same identifier; one bad line must not hide the good ones. +func parseScheduleRunRecords(stdout string) []ScheduleRunRecord { + var out []ScheduleRunRecord + for _, line := range strings.Split(stdout, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "{") { + continue + } + var record ScheduleRunRecord + if err := json.Unmarshal([]byte(line), &record); err != nil { + continue + } + out = append(out, record) + } + return out +} + +func (e *Engine) scheduledJob(name string) (app.ScheduledJob, error) { + jobs, err := e.Spec.ScheduledJobs() + if err != nil { + return app.ScheduledJob{}, err + } + for _, job := range jobs { + if job.Name == name { + return job, nil + } + } + return app.ScheduledJob{}, fmt.Errorf("job %q is not a scheduled job", name) +} + +// ScheduleHistory returns the newest n run records of one job, newest first. +func (e *Engine) ScheduleHistory(ctx context.Context, name string, n int) ([]ScheduleRunRecord, error) { + job, err := e.scheduledJob(name) + if err != nil { + return nil, err + } + res, err := e.T.Run(ctx, scheduleHistoryCommand(e.names().ScheduledJobUnit(job.Name), n)) + if err != nil { + return nil, err + } + records := parseScheduleRunRecords(res.Stdout) + if records == nil { + records = []ScheduleRunRecord{} + } + return records, nil +} + +// ScheduleList reads every declared job's timer in one round trip. +func (e *Engine) ScheduleList(ctx context.Context) ([]ScheduleListing, error) { + jobs, err := e.Spec.ScheduledJobs() + if err != nil { + return nil, err + } + if len(jobs) == 0 { + return []ScheduleListing{}, nil + } + var commands []string + for _, job := range jobs { + unit := e.names().ScheduledJobUnit(job.Name) + commands = append(commands, + "printf '%s\\n' "+q("@@"+job.Name), + "systemctl show "+q(unit+".timer")+" --no-pager --property=ActiveState --property=NextElapseUSecRealtime --property=LastTriggerUSec 2>/dev/null || true") + } + res, err := e.T.Run(ctx, strings.Join(commands, "\n")) + if err != nil { + return nil, err + } + observed := map[string]map[string]string{} + current := "" + for _, line := range strings.Split(res.Stdout, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "@@") { + current = strings.TrimPrefix(line, "@@") + observed[current] = map[string]string{} + continue + } + if key, value, ok := strings.Cut(line, "="); ok && current != "" { + observed[current][key] = value + } + } + out := make([]ScheduleListing, 0, len(jobs)) + for _, job := range jobs { + values := observed[job.Name] + out = append(out, ScheduleListing{ + Name: job.Name, Unit: e.names().ScheduledJobUnit(job.Name), Cron: job.Cron, Timezone: job.Timezone, + DeployLock: job.DeployLock, Timeout: job.Timeout, TimerState: values["ActiveState"], + NextRun: values["NextElapseUSecRealtime"], LastTrigger: values["LastTriggerUSec"], + }) + } + return out, nil +} + +// ScheduleLogs streams the journal of one run. The run id is systemd's +// invocation id, so the output is exactly that activation and nothing else. +// With no run given, the newest record's run is used; with no record at all, +// the unit's recent log stands in. +func (e *Engine) ScheduleLogs(ctx context.Context, name, run string, tail int, stdout, stderr io.Writer) error { + job, err := e.scheduledJob(name) + if err != nil { + return err + } + unit := e.names().ScheduledJobUnit(job.Name) + if run == "" { + records, err := e.ScheduleHistory(ctx, name, 1) + if err != nil { + return err + } + if len(records) > 0 { + run = records[0].Run + } + } + if tail <= 0 { + tail = 200 + } + var cmd string + switch { + case run == "": + cmd = "journalctl -u " + q(unit+".service") + " -n " + strconv.Itoa(tail) + " --no-pager -o short-iso" + case scheduleRunID.MatchString(run): + cmd = "journalctl _SYSTEMD_INVOCATION_ID=" + run + " --no-pager -o short-iso" + default: + return fmt.Errorf("run id %q is not a systemd invocation id", run) + } + return e.T.RunStream(ctx, cmd, stdout, stderr) +} diff --git a/internal/engine/schedule_history_test.go b/internal/engine/schedule_history_test.go new file mode 100644 index 0000000..5777c23 --- /dev/null +++ b/internal/engine/schedule_history_test.go @@ -0,0 +1,106 @@ +package engine + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/transport" +) + +const sampleRunRecords = `{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"manual","operation":"20260905-151200-schedule_run-7c1e","release":"20260905-140000-ab12cd","started_at":"2026-09-05T15:12:01Z","finished_at":"2026-09-05T15:12:31Z","duration_s":30,"attempts":2,"exit_status":0,"outcome":"success","inputs":{"SOURCE":"prices"}} +not json +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","operation":"","release":"20260905-140000-ab12cd","started_at":"2026-09-05T15:00:01Z","finished_at":"2026-09-05T15:00:02Z","duration_s":1,"attempts":0,"exit_status":75,"outcome":"skipped","inputs":{}} +` + +func TestParseScheduleRunRecordsSkipsNoiseAndKeepsOrder(t *testing.T) { + records := parseScheduleRunRecords(sampleRunRecords) + if len(records) != 2 { + t.Fatalf("records = %#v", records) + } + if records[0].Run != "b2c3d4e5f60718293a4b5c6d7e8f9012" || records[0].Trigger != "manual" || records[0].Attempts != 2 || + records[0].DurationSeconds != 30 || records[0].Inputs["SOURCE"] != "prices" || records[0].ExitStatus == nil || *records[0].ExitStatus != 0 { + t.Fatalf("first record was not decoded: %#v", records[0]) + } + if records[1].Outcome != "skipped" || *records[1].ExitStatus != 75 { + t.Fatalf("second record was not decoded: %#v", records[1]) + } +} + +func scheduledFixture(t *testing.T) (*Engine, *transport.Fake) { + t.Helper() + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{} + return New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}), f +} + +func TestScheduleHistoryReadsTheUnitJournalNewestFirst(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "journalctl") { + return transport.Result{Stdout: sampleRunRecords}, true + } + return transport.Result{}, false + } + records, err := e.ScheduleHistory(context.Background(), "nightly", 20) + if err != nil { + t.Fatal(err) + } + if len(records) != 2 || records[0].Outcome != "success" { + t.Fatalf("records = %#v", records) + } + seq := strings.Join(f.Commands, "\n") + for _, want := range []string{"journalctl -u 'ob-sample-nightly.service'", "-t ob-run", "-o cat", "-r", "-n 20", "--no-pager"} { + if !strings.Contains(seq, want) { + t.Fatalf("history read is missing %q:\n%s", want, seq) + } + } + if _, err := e.ScheduleHistory(context.Background(), "web", 20); err == nil { + t.Fatal("history of a non-scheduled workload was not refused") + } +} + +func TestScheduleListReadsTimerState(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: "@@nightly\nActiveState=active\nNextElapseUSecRealtime=Sat 2026-09-06 02:00:00 UTC\nLastTriggerUSec=Fri 2026-09-05 02:00:00 UTC\n"}, true + } + return transport.Result{}, false + } + listing, err := e.ScheduleList(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(listing) != 1 || listing[0].Unit != "ob-sample-nightly" || listing[0].TimerState != "active" || + listing[0].NextRun != "Sat 2026-09-06 02:00:00 UTC" || listing[0].LastTrigger != "Fri 2026-09-05 02:00:00 UTC" || listing[0].Cron != "0 2 * * *" { + t.Fatalf("listing = %#v", listing) + } +} + +func TestScheduleLogsTargetsOneInvocation(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "-t ob-run") { + return transport.Result{Stdout: sampleRunRecords}, true + } + return transport.Result{}, false + } + var out bytes.Buffer + if err := e.ScheduleLogs(context.Background(), "nightly", "", 200, &out, &out); err != nil { + t.Fatal(err) + } + seq := strings.Join(f.Commands, "\n") + if !strings.Contains(seq, "journalctl _SYSTEMD_INVOCATION_ID=b2c3d4e5f60718293a4b5c6d7e8f9012") { + t.Fatalf("logs did not target the newest run:\n%s", seq) + } + if err := e.ScheduleLogs(context.Background(), "nightly", "../etc", 200, &out, &out); err == nil { + t.Fatal("an invalid run id reached the shell") + } +} From 98e559d45a89707c63dc7f20a523b970423acd4f Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:14:48 -0700 Subject: [PATCH 04/25] feat(status): read scheduled-run outcomes, attempts and next elapse from the journal Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule_status.go | 99 ++++++++++++++++++++++++++---- internal/engine/schedule_test.go | 88 ++++++++++++++++++++++++++ internal/engine/status.go | 17 ++++- 3 files changed, 189 insertions(+), 15 deletions(-) diff --git a/internal/engine/schedule_status.go b/internal/engine/schedule_status.go index f210113..24d38e0 100644 --- a/internal/engine/schedule_status.go +++ b/internal/engine/schedule_status.go @@ -12,7 +12,9 @@ import ( // StatusSchedule is the host-observed state of one declared scheduled job. // systemd keeps Result after a oneshot exits, so a failed or timed-out run stays -// visible until a later successful run clears it. +// visible until a later successful run clears it. The run records the notifier +// writes to the journal say more: outcome, attempts, duration, and how many +// firings in a row have failed. type StatusSchedule struct { Name string `json:"name"` Unit string `json:"unit"` @@ -26,6 +28,17 @@ type StatusSchedule struct { LastExitStatus int `json:"last_exit_status,omitempty"` Diverged bool `json:"diverged"` Issues []string `json:"issues,omitempty"` + + // From the timer and the run records. + NextRun string `json:"next_run,omitempty"` + Attempt int `json:"attempt,omitempty"` + LastOutcome string `json:"last_outcome,omitempty"` + LastDurationSeconds int `json:"last_duration_s,omitempty"` + LastAttempts int `json:"last_attempts,omitempty"` + ConsecutiveFailures int `json:"consecutive_failures,omitempty"` + // JournalPersistent is false when the host keeps its journal in memory, so + // the records above only reach back to the last boot. + JournalPersistent bool `json:"journal_persistent"` } type scheduleUnitObservation struct { @@ -35,6 +48,9 @@ type scheduleUnitObservation struct { exitStatus int release string startedAt string + attempt string + next string + history []ScheduleRunRecord } func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) { @@ -46,16 +62,21 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) return []StatusSchedule{}, nil } - var commands []string + commands := []string{ + "printf '%s\\n' '@@journal'", + "if [ -d /var/log/journal ]; then echo persistent; else echo volatile; fi", + } for _, job := range jobs { unit := e.names().ScheduledJobUnit(job.Name) commands = append(commands, "printf '%s\\n' "+q("@@"+job.Name+":service"), "systemctl show "+q(unit+".service")+" --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus", "printf '%s\\n' "+q("@@"+job.Name+":timer"), - "systemctl show "+q(unit+".timer")+" --no-pager --property=LoadState --property=ActiveState", + "systemctl show "+q(unit+".timer")+" --no-pager --property=LoadState --property=ActiveState --property=NextElapseUSecRealtime", "printf '%s\\n' "+q("@@"+job.Name+":run"), "cat "+q(e.names().ScheduledJobRunState(job.Name))+" 2>/dev/null || true", + "printf '%s\\n' "+q("@@"+job.Name+":history"), + scheduleHistoryCommand(unit, 20), ) } res, err := e.T.Run(ctx, strings.Join(commands, "\n")) @@ -67,8 +88,10 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) } observed := map[string]map[string]scheduleUnitObservation{} + journalPersistent := true name, kind := "", "" values := map[string]string{} + var raw []string flush := func() { if name == "" || kind == "" { return @@ -80,18 +103,35 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) observed[name][kind] = scheduleUnitObservation{ loadState: values["LoadState"], activeState: values["ActiveState"], result: values["Result"], exitStatus: exit, - release: values["release"], startedAt: values["started_at"], + release: values["release"], startedAt: values["started_at"], attempt: values["attempt"], + next: values["NextElapseUSecRealtime"], + history: parseScheduleRunRecords(strings.Join(raw, "\n")), } values = map[string]string{} + raw = nil } for _, line := range strings.Split(res.Stdout, "\n") { line = strings.TrimSpace(line) if strings.HasPrefix(line, "@@") { flush() marker := strings.TrimPrefix(line, "@@") + if marker == "journal" { + name, kind = "", "" + continue + } name, kind, _ = strings.Cut(marker, ":") continue } + if name == "" { + if line == "volatile" { + journalPersistent = false + } + continue + } + if kind == "history" { + raw = append(raw, line) + continue + } if key, value, ok := strings.Cut(line, "="); ok { values[key] = value } @@ -104,18 +144,40 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) service := observed[job.Name]["service"] timer := observed[job.Name]["timer"] run := observed[job.Name]["run"] + records := observed[job.Name]["history"].history status := StatusSchedule{ Name: job.Name, Unit: unit, TimerState: timer.activeState, Running: service.activeState == "activating", DeployLock: job.DeployLock, Timeout: job.Timeout, LastResult: service.result, LastExitStatus: service.exitStatus, + NextRun: timer.next, JournalPersistent: journalPersistent, + } + if status.Running { + status.Attempt, _ = strconv.Atoi(run.attempt) + if job.DeployLock == "pinned" { + _, timeErr := time.Parse(time.RFC3339, run.startedAt) + if !release.IsID(run.release) || timeErr != nil { + status.Issues = append(status.Issues, "running pinned job state is unavailable or invalid") + } else { + status.PinnedRelease = run.release + status.StartedAt = run.startedAt + } + } } - if status.Running && job.DeployLock == "pinned" { - _, timeErr := time.Parse(time.RFC3339, run.startedAt) - if !release.IsID(run.release) || timeErr != nil { - status.Issues = append(status.Issues, "running pinned job state is unavailable or invalid") - } else { - status.PinnedRelease = run.release - status.StartedAt = run.startedAt + if len(records) > 0 { + last := records[0] + status.LastOutcome = last.Outcome + status.LastDurationSeconds = last.DurationSeconds + status.LastAttempts = last.Attempts + // A skip says nothing about the job, so it neither breaks nor + // extends a failure streak. + for _, record := range records { + if record.Outcome == "skipped" { + continue + } + if record.Outcome != "failure" && record.Outcome != "timeout" { + break + } + status.ConsecutiveFailures++ } } if timer.loadState != "loaded" || timer.activeState != "active" { @@ -124,8 +186,19 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) if service.loadState != "loaded" { status.Issues = append(status.Issues, "service unit is not loaded") } - if service.result != "" && service.result != "success" { - status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %d)", service.result, service.exitStatus)) + switch { + case status.LastOutcome == "failure" || status.LastOutcome == "timeout": + exit := "?" + if records[0].ExitStatus != nil { + exit = strconv.Itoa(*records[0].ExitStatus) + } + status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %s)", status.LastOutcome, exit)) + case status.LastOutcome == "": + // No record yet: an older runner, or a journal that did not keep + // it. systemd's own result is the next best witness. + if service.result != "" && service.result != "success" { + status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %d)", service.result, service.exitStatus)) + } } status.Diverged = len(status.Issues) > 0 statuses = append(statuses, status) diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index 6b30283..14e341d 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -991,3 +991,91 @@ func TestScheduledJobNotifierRecordsEachOutcomeAndRemovesState(t *testing.T) { }) } } + +func TestScheduleStatusPrefersTheRunRecordOverSystemdResult(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: `@@journal +persistent +@@nightly:service +LoadState=loaded +ActiveState=inactive +Result=success +ExecMainStatus=75 +@@nightly:timer +LoadState=loaded +ActiveState=active +NextElapseUSecRealtime=Sat 2026-09-06 02:00:00 UTC +@@nightly:run +@@nightly:history +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","release":"r1","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:02Z","duration_s":1,"attempts":0,"exit_status":75,"outcome":"skipped","inputs":{}} +{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"timer","release":"r1","started_at":"2026-09-04T02:00:01Z","finished_at":"2026-09-04T02:05:02Z","duration_s":301,"attempts":3,"exit_status":1,"outcome":"failure","inputs":{}} +{"run":"c3d4e5f60718293a4b5c6d7e8f901234","job":"nightly","trigger":"timer","release":"r1","started_at":"2026-09-03T02:00:01Z","finished_at":"2026-09-03T02:01:02Z","duration_s":61,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}} +`}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + got := statuses[0] + if got.LastOutcome != "skipped" || got.NextRun != "Sat 2026-09-06 02:00:00 UTC" || !got.JournalPersistent { + t.Fatalf("record fields not surfaced: %#v", got) + } + // The newest record is a skip: it neither counts as a failure nor clears + // the failure before it, and it raises no issue of its own. + if got.ConsecutiveFailures != 1 || got.Diverged { + t.Fatalf("a skip neither counts as nor clears a failure: %#v", got) + } + if got.LastAttempts != 0 || got.LastDurationSeconds != 1 { + t.Fatalf("last run detail not surfaced: %#v", got) + } +} + +func TestScheduleStatusCountsConsecutiveFailuresFromRecords(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: `@@journal +volatile +@@nightly:service +LoadState=loaded +ActiveState=failed +Result=exit-code +ExecMainStatus=1 +@@nightly:timer +LoadState=loaded +ActiveState=active +@@nightly:run +@@nightly:history +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:02Z","duration_s":1,"attempts":2,"exit_status":1,"outcome":"failure","inputs":{}} +{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"timer","started_at":"2026-09-04T02:00:01Z","finished_at":"2026-09-04T02:05:02Z","duration_s":301,"attempts":2,"exit_status":null,"outcome":"timeout","inputs":{}} +{"run":"c3d4e5f60718293a4b5c6d7e8f901234","job":"nightly","trigger":"timer","started_at":"2026-09-03T02:00:01Z","finished_at":"2026-09-03T02:01:02Z","duration_s":61,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}} +`}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + got := statuses[0] + if got.ConsecutiveFailures != 2 || !got.Diverged || got.JournalPersistent { + t.Fatalf("failures not counted: %#v", got) + } + if !strings.Contains(strings.Join(got.Issues, "; "), "last run failed: failure (exit 1)") { + t.Fatalf("issue does not name the outcome: %#v", got.Issues) + } +} diff --git a/internal/engine/status.go b/internal/engine/status.go index def1fe4..415c003 100644 --- a/internal/engine/status.go +++ b/internal/engine/status.go @@ -177,14 +177,27 @@ func (e *Engine) Status(ctx context.Context) error { } if schedule.Running { detail := fmt.Sprintf("running; policy: %s; timeout: %s", schedule.DeployLock, schedule.Timeout) + if schedule.Attempt > 0 { + detail += fmt.Sprintf("; attempt: %d", schedule.Attempt) + } if schedule.PinnedRelease != "" { detail += fmt.Sprintf("; release: %s; started: %s", schedule.PinnedRelease, schedule.StartedAt) } fmt.Fprintf(e.Opts.Out, "schedule %-11s %s\n", schedule.Name, detail) continue } - fmt.Fprintf(e.Opts.Out, "schedule %-11s active; policy: %s; timeout: %s; last: %s\n", - schedule.Name, schedule.DeployLock, schedule.Timeout, result) + detail := fmt.Sprintf("active; policy: %s; timeout: %s", schedule.DeployLock, schedule.Timeout) + if schedule.NextRun != "" { + detail += "; next: " + schedule.NextRun + } + if schedule.LastOutcome != "" { + result = fmt.Sprintf("%s (%ds, %d attempt(s))", schedule.LastOutcome, schedule.LastDurationSeconds, schedule.LastAttempts) + } + detail += "; last: " + result + if !schedule.JournalPersistent { + detail += "; journal: volatile, history since boot only" + } + fmt.Fprintf(e.Opts.Out, "schedule %-11s %s\n", schedule.Name, detail) } if managed { From 5fa60a8578a03a197888d17da4c9a4e6199ac4c2 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:16:27 -0700 Subject: [PATCH 05/25] docs: implementation plan for scheduled-job runs (#155) Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- .../plans/2026-09-05-scheduled-job-runs.md | 2484 +++++++++++++++++ 1 file changed, 2484 insertions(+) create mode 100644 docs/superpowers/plans/2026-09-05-scheduled-job-runs.md diff --git a/docs/superpowers/plans/2026-09-05-scheduled-job-runs.md b/docs/superpowers/plans/2026-09-05-scheduled-job-runs.md new file mode 100644 index 0000000..99c3c88 --- /dev/null +++ b/docs/superpowers/plans/2026-09-05-scheduled-job-runs.md @@ -0,0 +1,2484 @@ +# Scheduled Job Runs Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give scheduled jobs a per-run record in the host journal, bounded retry with backoff, per-outcome notifications, and declared inputs with an operator-initiated `ob schedule run`, without any resident process or new store on the host. + +**Architecture:** The generated runner script gains a state file, a retry loop and manual-input handling; the generated `ExecStopPost` notifier finalises one JSON record per run into the journal via `systemd-cat -t ob-run` and sends notifications per outcome. `ob` reads records back with `journalctl` (`schedule history`, `schedule logs`, `schedule list`, richer `status`) and starts a unit with validated inputs (`schedule run`), journaled as `schedule_run`. Every new project-file field is optional and every default preserves today's behaviour. + +**Tech Stack:** Go 1.27, cobra CLI, POSIX sh rendered from Go, systemd (timers, `INVOCATION_ID`, `TRIGGER_UNIT`, `SuccessExitStatus`, `systemd-cat`, `journalctl`), `transport.Fake` for unit tests, Lima Ubuntu 24.04 for `just server-e2e`. + +**Spec:** https://github.com/labstack/onebox/issues/155 (revised body, 2026-09-05) + +## Global Constraints + +- Onebox environment variables use the `ONEBOX_` prefix. `OB_` is retired and `just env-namespace` fails on any tracked-file match of `\bOB_[A-Z0-9_]+`. Reserved input prefix and metadata line are therefore `ONEBOX_`. +- Workload names match `^[a-z]([a-z0-9-]{0,38}[a-z0-9])?$`. +- Input names match `^[A-Z][A-Z0-9_]*$`, must not start with `ONEBOX_`, must not equal a key in the workload's `env`. +- Input values (defaults and overrides): at most 256 bytes, no `"`, no `\`, no control characters (`0x00-0x1f`, `0x7f`). Patterns match the whole value. +- `retry.attempts` 1..10 default 1; `retry.backoff` default `30s`; `retry.max_backoff` default `10m`; worst-case total backoff must be strictly less than `schedule.timeout`. +- `schedule.notify` values: `success`, `failure`, `timeout`, `skipped`; default `[failure, timeout]`. +- `inputs` requires `schedule` and `data_effect: none`. `ob schedule run` refuses any other data effect. +- Service unit gains `SuccessExitStatus=75`. +- Run record syslog identifier: `ob-run`. Record fields: `run, job, trigger, operation, release, started_at, finished_at, duration_s, attempts, exit_status, outcome, inputs`. +- Manual trigger detection: `$TRIGGER_UNIT` unset. Host needs systemd 252+ only when a job declares `inputs`; checked in `SyncSchedules` beside the existing `systemd-analyze calendar` check. +- Commit messages end with `Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5`. +- Gates before every commit: `go build ./... && go vet ./... && go test ./...`. Before the final commit of each slice also `just docs-generate`, `go run ./cmd/ob schema --out docs/onebox.run-v1.schema.json`, `just docs-generate-check`, `just env-namespace`. + +--- + +## Slice 1: History + +### Task 1: State file in both runner modes, `SuccessExitStatus=75` + +**Files:** +- Modify: `internal/engine/schedule.go:172-245` (`scheduleRunnerScript`, `pinnedScheduleRunnerScript`), `internal/engine/schedule.go:262-280` (`scheduleServiceUnit`) +- Test: `internal/engine/schedule_test.go` + +**Interfaces:** +- Produces: shell function `write_state ` defined in both runners, writing `release=`, `started_at=`, `started_epoch=`, `trigger=`, `operation=`, `attempt=`, `inputs=` lines atomically to `names.ScheduledJobRunState(job)`. Runner no longer deletes the state file. Go helper `scheduleStateFunction(state string) []string`. + +- [ ] **Step 1: Write the failing test** + +Append to `internal/engine/schedule_test.go`: + +```go +func TestScheduledJobRunnersRecordRunStateForTheNotifier(t *testing.T) { + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + for _, tc := range []struct { + name string + job app.ScheduledJob + }{ + {"exclusive", app.ScheduledJob{Name: "nightly", Cron: "0 2 * * *", Timezone: "UTC", Calendar: "*-*-* 02:00:00", Timeout: "45m", DeployLock: "exclusive"}}, + {"pinned", app.ScheduledJob{Name: "nightly", Cron: "0 2 * * *", Timezone: "UTC", Calendar: "*-*-* 02:00:00", Timeout: "45m", DeployLock: "pinned"}}, + } { + t.Run(tc.name, func(t *testing.T) { + runner := scheduleRunnerScript("sample", tc.job, names, "/var/lib/ob/sample/lock", nil) + for _, want := range []string{ + "state='/var/lib/ob/sample/schedule/nightly.state'", + "write_state() {", + "started_epoch=%s", + `trigger=%s`, + "write_state 1", + "mv -f \"$tmp\" \"$state\"", + } { + if !strings.Contains(runner, want) { + t.Errorf("%s runner is missing %q:\n%s", tc.name, want, runner) + } + } + if strings.Contains(runner, `rm -f "$state"`) { + t.Errorf("%s runner removes the state the notifier finalises:\n%s", tc.name, runner) + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(runner) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("%s runner is not valid POSIX shell: %v: %s\n%s", tc.name, err, output, runner) + } + }) + } + service := scheduleServiceUnit("sample", app.ScheduledJob{Name: "nightly", Timeout: "45m"}, + "/etc/systemd/system/ob-sample-nightly.run", "/etc/systemd/system/ob-sample-nightly.notify") + if !strings.Contains(service, "SuccessExitStatus=75") { + t.Errorf("a lock-conflict skip must not be a failed unit:\n%s", service) + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `go test ./internal/engine -run TestScheduledJobRunnersRecordRunStateForTheNotifier` +Expected: FAIL, runner is missing `write_state() {`. + +- [ ] **Step 3: Implement** + +In `internal/engine/schedule.go` add after `scheduleContainerCleanup`: + +```go +// scheduleStateFunction renders the shell function both runners use to record +// the run in progress. The notifier reads it after the run ends, so the runner +// never removes it: a runner that cleaned up its own state would erase the +// only evidence a timed-out run leaves behind. +func scheduleStateFunction() []string { + return []string{ + "write_state() {", + " umask 077", + " printf 'release=%s\\nstarted_at=%s\\nstarted_epoch=%s\\ntrigger=%s\\noperation=%s\\nattempt=%s\\ninputs=%s\\n' " + + "\"$release\" \"$started_at\" \"$started_epoch\" \"$trigger\" \"$operation\" \"$1\" \"$inputs_json\" >\"$tmp\"", + " mv -f \"$tmp\" \"$state\"", + "}", + } +} + +// scheduleRunPreamble sets the variables write_state records. The trigger is +// systemd's own word for it: a timer activation carries TRIGGER_UNIT (systemd +// 252+), anything else is an operator. +func scheduleRunPreamble(state string) []string { + return append([]string{ + "state=" + q(state), + "tmp=\"$state.$$\"", + "started_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", + "started_epoch=$(date -u '+%s')", + "if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=manual; fi", + "operation=''", + "inputs_json=''", + }, scheduleStateFunction()...) +} +``` + +Rewrite the exclusive runner body (replace lines from `scheduleContainerCleanup(container),` through `compose,` in `scheduleRunnerScript`): + +```go + lines := []string{ + "#!/bin/sh", + "# Written by Onebox. Edits are overwritten on the next deploy.", + "set -eu", + "install -d -m 700 " + q(names.AppDir()+"/schedule"), + "exec 9>" + q(names.ScheduledJobRunLock(job.Name)), + "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 9", + "exec 8>" + q(names.ScheduleRunLock()), + "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 8", + "if [ -e " + q(applicationLock) + " ]; then echo 'onebox: an application operation holds the deploy lock' >&2; exit 75; fi", + "release_dir=$(readlink -f " + q(names.CurrentLink()) + " 2>/dev/null || true)", + "release=${release_dir##*/}", + scheduleContainerCleanup(container), + "cleanup() { " + scheduleContainerCleanup(container) + "; rm -f \"$tmp\"; }", + "trap cleanup 0", + "trap 'exit 129' 1", + "trap 'exit 130' 2", + "trap 'exit 143' 15", + } + lines = append(lines, scheduleRunPreamble(names.ScheduledJobRunState(job.Name))...) + lines = append(lines, "write_state 1", compose, "") + return strings.Join(lines, "\n") +``` + +In `pinnedScheduleRunnerScript` replace the block from `"state=" + q(state),` through `"mv -f \"$tmp\" \"$state\"",` with: + +```go + "cleanup() { " + scheduleContainerCleanup(container) + "; rm -f \"$tmp\"; }", + "trap cleanup 0", + "trap 'exit 129' 1", + "trap 'exit 130' 2", + "trap 'exit 143' 15", + } + lines = append(lines, scheduleRunPreamble(state)...) + lines = append(lines, "write_state 1", "/usr/bin/flock --unlock 8", compose, "") + return strings.Join(lines, "\n") +``` + +Remove the now-unused `"started_at=$(date ...)"`, `"umask 077"`, `"printf 'release=...'"` lines and the old `"state="`/`"tmp="` lines from the pinned runner. Keep `"release=${release_dir##*/}"` in the pinned runner (it already exists). + +In `scheduleServiceUnit` add after `"TimeoutStartSec=" + job.Timeout,`: + +```go + // Exit 75 is the runner's "skipped for a lock conflict". It is a fact + // about timing, not a failure of the job, and it must not leave the + // unit failed or trip failure notifications. + "SuccessExitStatus=75", +``` + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine` +Expected: PASS. `TestScheduleStatusReportsRunningPinnedRelease` still passes because the status reader only needs `release=` and `started_at=` lines. + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine/schedule.go internal/engine/schedule_test.go +git commit -m "feat(schedule): record run state in both runner modes and stop treating a skip as failure + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 2: Notifier finalises one journal record per run + +**Files:** +- Modify: `internal/engine/schedule.go:287-335` (`scheduleFailureNotifier`) +- Test: `internal/engine/schedule_test.go` + +**Interfaces:** +- Produces: the notifier script reads the state file, derives `outcome`, writes one JSON line via `systemd-cat -t ob-run`, deletes the state file, then keeps today's failure/timeout notification behaviour. Constant `scheduleRunIdentifier = "ob-run"`. + +- [ ] **Step 1: Write the failing test** + +```go +func TestScheduledJobNotifierWritesOneRunRecordToTheJournal(t *testing.T) { + cfg := testConfig() + f := &transport.Fake{TargetName: "root@example.internal"} + e := New(cfg, testProject(t), f, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + script, err := e.scheduleFailureNotifier("nightly") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "state='/var/lib/ob/sample/schedule/nightly.state'", + `started_epoch=*) started_epoch=${line#started_epoch=}`, + `rm -f "$state"`, + `result=${SERVICE_RESULT:-success}`, + `status=${EXIT_STATUS:-0}`, + `outcome=timeout`, + `outcome=skipped`, + `outcome=success`, + `outcome=failure`, + `"run":"%s","job":"%s","trigger":"%s","operation":"%s","release":"%s"`, + `"duration_s":%s,"attempts":%s,"exit_status":%s,"outcome":"%s","inputs":{%s}`, + `"${INVOCATION_ID:-}" 'nightly'`, + "systemd-cat -t ob-run", + } { + if !strings.Contains(script, want) { + t.Errorf("notifier is missing %q:\n%s", want, script) + } + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(script) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("notifier is not valid POSIX shell: %v: %s\n%s", err, output, script) + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `go test ./internal/engine -run TestScheduledJobNotifierWritesOneRunRecordToTheJournal` +Expected: FAIL, missing `state='...'`. + +- [ ] **Step 3: Implement** + +Add the constant and a record-rendering helper near `scheduleNotificationTimestamp`: + +```go +// scheduleRunIdentifier is the syslog identifier of the one line the notifier +// writes per run. `journalctl -u -t ob-run` is the run history. +const scheduleRunIdentifier = "ob-run" + +// scheduleRunRecordLines finalises the run the runner started. It runs in +// ExecStopPost because only systemd knows how the run ended: a timed-out +// runner is killed mid-sleep and cannot write its own outcome. +func scheduleRunRecordLines(job, state string) []string { + return []string{ + "state=" + q(state), + "release=''; started_at=''; started_epoch=''; trigger=''; operation=''; attempt=0; inputs=''", + "if [ -f \"$state\" ]; then", + " while IFS= read -r line || [ -n \"$line\" ]; do", + " case \"$line\" in", + " release=*) release=${line#release=} ;;", + " started_at=*) started_at=${line#started_at=} ;;", + " started_epoch=*) started_epoch=${line#started_epoch=} ;;", + " trigger=*) trigger=${line#trigger=} ;;", + " operation=*) operation=${line#operation=} ;;", + " attempt=*) attempt=${line#attempt=} ;;", + " inputs=*) inputs=${line#inputs=} ;;", + " esac", + " done <\"$state\"", + " rm -f \"$state\"", + "fi", + "if [ -z \"$trigger\" ]; then if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=manual; fi; fi", + "result=${SERVICE_RESULT:-success}", + "status=${EXIT_STATUS:-0}", + "case \"$status\" in ''|*[!0-9]*) status=null ;; esac", + "case \"$attempt\" in ''|*[!0-9]*) attempt=0 ;; esac", + "if [ \"$result\" = timeout ]; then outcome=timeout", + "elif [ \"$status\" = 75 ]; then outcome=skipped", + "elif [ \"$result\" = success ] && [ \"$status\" = 0 ]; then outcome=success", + "else outcome=failure; fi", + "finished_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", + "now=$(date -u '+%s')", + "duration=0", + "case \"$started_epoch\" in ''|*[!0-9]*) ;; *) duration=$((now - started_epoch)) ;; esac", + "[ -z \"$started_at\" ] && started_at=$finished_at", + "printf '{\"run\":\"%s\",\"job\":\"%s\",\"trigger\":\"%s\",\"operation\":\"%s\",\"release\":\"%s\",\"started_at\":\"%s\",\"finished_at\":\"%s\",\"duration_s\":%s,\"attempts\":%s,\"exit_status\":%s,\"outcome\":\"%s\",\"inputs\":{%s}}\\n' " + + "\"${INVOCATION_ID:-}\" " + q(job) + " \"$trigger\" \"$operation\" \"$release\" \"$started_at\" \"$finished_at\" \"$duration\" \"$attempt\" \"$status\" \"$outcome\" \"$inputs\" " + + "| systemd-cat -t " + scheduleRunIdentifier + " || true", + } +} +``` + +In `scheduleFailureNotifier`, replace the line `` `[ "${SERVICE_RESULT:-success}" = success ] && exit 0`, `` with: + +```go + } + lines = append(lines, scheduleRunRecordLines(job, e.names().ScheduledJobRunState(job))...) + lines = append(lines, `case "$outcome" in failure|timeout) ;; *) exit 0 ;; esac`) + lines = append(lines, "") + if false { +``` + +Concretely the function becomes: + +```go + lines := []string{ + "#!/bin/sh", + "# Written by Onebox. Edits are overwritten on the next deploy.", + "set -u", + "exec 9>" + q(e.names().ScheduledJobRunLock(job)), + "if /usr/bin/flock --exclusive --nonblock 9; then", + " " + scheduleContainerCleanup(e.names().Container(job, 1)), + "fi", + } + lines = append(lines, scheduleRunRecordLines(job, e.names().ScheduledJobRunState(job))...) + lines = append(lines, `case "$outcome" in failure|timeout) ;; *) exit 0 ;; esac`) + var sends []string + // ... unchanged send rendering ... +``` + +Update `TestScheduledJobFailureNotifierUsesConfiguredWebhooks`: replace the expectation `` `${SERVICE_RESULT:-success}` `` with `` `result=${SERVICE_RESULT:-success}` ``. + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine/schedule.go internal/engine/schedule_test.go +git commit -m "feat(schedule): finalise one journal record per scheduled run from ExecStopPost + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 3: Read run records back: `ScheduleHistory`, `ScheduleList`, run logs + +**Files:** +- Create: `internal/engine/schedule_history.go` +- Test: `internal/engine/schedule_history_test.go` + +**Interfaces:** +- Produces: + - `type ScheduleRunRecord struct { Run, Job, Trigger, Operation, Release, StartedAt, FinishedAt string; DurationSeconds int; Attempts int; ExitStatus *int; Outcome string; Inputs map[string]string }` with JSON tags `run, job, trigger, operation, release, started_at, finished_at, duration_s, attempts, exit_status, outcome, inputs`. + - `func parseScheduleRunRecords(stdout string) []ScheduleRunRecord` (skips unparseable lines). + - `func scheduleHistoryCommand(unit string, n int) string` = `journalctl -u .service -t ob-run -o cat -r -n --no-pager 2>/dev/null || true`. + - `func (e *Engine) ScheduleHistory(ctx, job string, n int) ([]ScheduleRunRecord, error)`; unknown or unscheduled job returns an error `job %q is not a scheduled job`. + - `type ScheduleListing struct { Name, Unit, Cron, Timezone, DeployLock, Timeout, TimerState, NextRun, LastTrigger string }` JSON tags `name, unit, cron, timezone, deploy_lock, timeout, timer_state, next_run, last_trigger`. + - `func (e *Engine) ScheduleList(ctx) ([]ScheduleListing, error)` using `systemctl show .timer --no-pager --property=ActiveState --property=NextElapseUSecRealtime --property=LastTriggerUSec` batched with `@@` markers. + - `func (e *Engine) ScheduleLogs(ctx, job, run string, tail int, stdout, stderr io.Writer) error`: with `run` empty, resolve the newest record's run id via `ScheduleHistory(ctx, job, 1)`; command `journalctl _SYSTEMD_INVOCATION_ID= --no-pager -o short-iso`; with no record fall back to `journalctl -u .service -n --no-pager -o short-iso`. Run ids are validated against `^[0-9a-f]{32}$` before reaching a shell. + +- [ ] **Step 1: Write the failing tests** + +`internal/engine/schedule_history_test.go`: + +```go +package engine + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/transport" +) + +const sampleRunRecords = `{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"manual","operation":"20260905-151200-schedule_run-7c1e","release":"20260905-140000-ab12cd","started_at":"2026-09-05T15:12:01Z","finished_at":"2026-09-05T15:12:31Z","duration_s":30,"attempts":2,"exit_status":0,"outcome":"success","inputs":{"SOURCE":"prices"}} +not json +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","operation":"","release":"20260905-140000-ab12cd","started_at":"2026-09-05T15:00:01Z","finished_at":"2026-09-05T15:00:02Z","duration_s":1,"attempts":0,"exit_status":75,"outcome":"skipped","inputs":{}} +` + +func TestParseScheduleRunRecordsSkipsNoiseAndKeepsOrder(t *testing.T) { + records := parseScheduleRunRecords(sampleRunRecords) + if len(records) != 2 { + t.Fatalf("records = %#v", records) + } + if records[0].Run != "b2c3d4e5f60718293a4b5c6d7e8f9012" || records[0].Trigger != "manual" || records[0].Attempts != 2 || + records[0].DurationSeconds != 30 || records[0].Inputs["SOURCE"] != "prices" || records[0].ExitStatus == nil || *records[0].ExitStatus != 0 { + t.Fatalf("first record was not decoded: %#v", records[0]) + } + if records[1].Outcome != "skipped" || *records[1].ExitStatus != 75 { + t.Fatalf("second record was not decoded: %#v", records[1]) + } +} + +func scheduledFixture(t *testing.T) (*Engine, *transport.Fake) { + t.Helper() + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{} + return New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}), f +} + +func TestScheduleHistoryReadsTheUnitJournalNewestFirst(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "journalctl") { + return transport.Result{Stdout: sampleRunRecords}, true + } + return transport.Result{}, false + } + records, err := e.ScheduleHistory(context.Background(), "nightly", 20) + if err != nil { + t.Fatal(err) + } + if len(records) != 2 || records[0].Outcome != "success" { + t.Fatalf("records = %#v", records) + } + seq := strings.Join(f.Commands, "\n") + for _, want := range []string{"journalctl -u ob-sample-nightly.service", "-t ob-run", "-o cat", "-r", "-n 20", "--no-pager"} { + if !strings.Contains(seq, want) { + t.Fatalf("history read is missing %q:\n%s", want, seq) + } + } + if _, err := e.ScheduleHistory(context.Background(), "web", 20); err == nil { + t.Fatal("history of a non-scheduled workload was not refused") + } +} + +func TestScheduleListReadsTimerState(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: "@@nightly\nActiveState=active\nNextElapseUSecRealtime=Sat 2026-09-06 02:00:00 UTC\nLastTriggerUSec=Fri 2026-09-05 02:00:00 UTC\n"}, true + } + return transport.Result{}, false + } + listing, err := e.ScheduleList(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(listing) != 1 || listing[0].Unit != "ob-sample-nightly" || listing[0].TimerState != "active" || + listing[0].NextRun != "Sat 2026-09-06 02:00:00 UTC" || listing[0].LastTrigger != "Fri 2026-09-05 02:00:00 UTC" || listing[0].Cron != "0 2 * * *" { + t.Fatalf("listing = %#v", listing) + } +} + +func TestScheduleLogsTargetsOneInvocation(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "-t ob-run") { + return transport.Result{Stdout: sampleRunRecords}, true + } + return transport.Result{}, false + } + var out bytes.Buffer + if err := e.ScheduleLogs(context.Background(), "nightly", "", 200, &out, &out); err != nil { + t.Fatal(err) + } + seq := strings.Join(f.Commands, "\n") + if !strings.Contains(seq, "journalctl _SYSTEMD_INVOCATION_ID=b2c3d4e5f60718293a4b5c6d7e8f9012") { + t.Fatalf("logs did not target the newest run:\n%s", seq) + } + if err := e.ScheduleLogs(context.Background(), "nightly", "../etc", 200, &out, &out); err == nil { + t.Fatal("an invalid run id reached the shell") + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/engine -run 'TestParseScheduleRunRecords|TestScheduleHistory|TestScheduleList|TestScheduleLogs'` +Expected: FAIL to compile (undefined symbols). + +- [ ] **Step 3: Implement `internal/engine/schedule_history.go`** + +```go +package engine + +import ( + "context" + "encoding/json" + "fmt" + "io" + "regexp" + "strconv" + "strings" + + "github.com/labstack/onebox/internal/app" +) + +// ScheduleRunRecord is the line the notifier writes to the journal when a +// scheduled run ends. The journal is the store: there is no file to trim and +// nothing that can disagree with the unit's own log. +type ScheduleRunRecord struct { + Run string `json:"run"` + Job string `json:"job"` + Trigger string `json:"trigger"` + Operation string `json:"operation,omitempty"` + Release string `json:"release,omitempty"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + DurationSeconds int `json:"duration_s"` + Attempts int `json:"attempts"` + ExitStatus *int `json:"exit_status"` + Outcome string `json:"outcome"` + Inputs map[string]string `json:"inputs,omitempty"` +} + +// ScheduleListing is one declared job beside its timer as the host reports it. +type ScheduleListing struct { + Name string `json:"name"` + Unit string `json:"unit"` + Cron string `json:"cron"` + Timezone string `json:"timezone"` + DeployLock string `json:"deploy_lock"` + Timeout string `json:"timeout"` + TimerState string `json:"timer_state"` + NextRun string `json:"next_run,omitempty"` + LastTrigger string `json:"last_trigger,omitempty"` +} + +var scheduleRunID = regexp.MustCompile(`^[0-9a-f]{32}$`) + +func scheduleHistoryCommand(unit string, n int) string { + if n <= 0 { + n = 20 + } + return "journalctl -u " + q(unit+".service") + " -t " + scheduleRunIdentifier + + " -o cat -r -n " + strconv.Itoa(n) + " --no-pager 2>/dev/null || true" +} + +func parseScheduleRunRecords(stdout string) []ScheduleRunRecord { + var out []ScheduleRunRecord + for _, line := range strings.Split(stdout, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "{") { + continue + } + var record ScheduleRunRecord + if err := json.Unmarshal([]byte(line), &record); err != nil { + continue + } + out = append(out, record) + } + return out +} + +func (e *Engine) scheduledJob(name string) (app.ScheduledJob, error) { + jobs, err := e.Spec.ScheduledJobs() + if err != nil { + return app.ScheduledJob{}, err + } + for _, job := range jobs { + if job.Name == name { + return job, nil + } + } + return app.ScheduledJob{}, fmt.Errorf("job %q is not a scheduled job", name) +} + +// ScheduleHistory returns the newest n run records of one job, newest first. +func (e *Engine) ScheduleHistory(ctx context.Context, name string, n int) ([]ScheduleRunRecord, error) { + job, err := e.scheduledJob(name) + if err != nil { + return nil, err + } + res, err := e.T.Run(ctx, scheduleHistoryCommand(e.names().ScheduledJobUnit(job.Name), n)) + if err != nil { + return nil, err + } + records := parseScheduleRunRecords(res.Stdout) + if records == nil { + records = []ScheduleRunRecord{} + } + return records, nil +} + +// ScheduleList reads every declared job's timer in one round trip. +func (e *Engine) ScheduleList(ctx context.Context) ([]ScheduleListing, error) { + jobs, err := e.Spec.ScheduledJobs() + if err != nil { + return nil, err + } + if len(jobs) == 0 { + return []ScheduleListing{}, nil + } + var commands []string + for _, job := range jobs { + unit := e.names().ScheduledJobUnit(job.Name) + commands = append(commands, + "printf '%s\\n' "+q("@@"+job.Name), + "systemctl show "+q(unit+".timer")+" --no-pager --property=ActiveState --property=NextElapseUSecRealtime --property=LastTriggerUSec 2>/dev/null || true") + } + res, err := e.T.Run(ctx, strings.Join(commands, "\n")) + if err != nil { + return nil, err + } + observed := map[string]map[string]string{} + current := "" + for _, line := range strings.Split(res.Stdout, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "@@") { + current = strings.TrimPrefix(line, "@@") + observed[current] = map[string]string{} + continue + } + if key, value, ok := strings.Cut(line, "="); ok && current != "" { + observed[current][key] = value + } + } + out := make([]ScheduleListing, 0, len(jobs)) + for _, job := range jobs { + values := observed[job.Name] + out = append(out, ScheduleListing{ + Name: job.Name, Unit: e.names().ScheduledJobUnit(job.Name), Cron: job.Cron, Timezone: job.Timezone, + DeployLock: job.DeployLock, Timeout: job.Timeout, TimerState: values["ActiveState"], + NextRun: values["NextElapseUSecRealtime"], LastTrigger: values["LastTriggerUSec"], + }) + } + return out, nil +} + +// ScheduleLogs streams the journal of one run. The run id is systemd's +// invocation id, so the output is exactly that activation and nothing else. +func (e *Engine) ScheduleLogs(ctx context.Context, name, run string, tail int, stdout, stderr io.Writer) error { + job, err := e.scheduledJob(name) + if err != nil { + return err + } + unit := e.names().ScheduledJobUnit(job.Name) + if run == "" { + records, err := e.ScheduleHistory(ctx, name, 1) + if err != nil { + return err + } + if len(records) > 0 { + run = records[0].Run + } + } + if tail <= 0 { + tail = 200 + } + var cmd string + switch { + case run == "": + cmd = "journalctl -u " + q(unit+".service") + " -n " + strconv.Itoa(tail) + " --no-pager -o short-iso" + case scheduleRunID.MatchString(run): + cmd = "journalctl _SYSTEMD_INVOCATION_ID=" + run + " --no-pager -o short-iso" + default: + return fmt.Errorf("run id %q is not a systemd invocation id", run) + } + return e.T.RunStream(ctx, cmd, stdout, stderr) +} +``` + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine` +Expected: PASS. If `transport.Fake` lacks `RunStream`, check `internal/transport/fake.go`; it implements the interface, so this compiles. + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine/schedule_history.go internal/engine/schedule_history_test.go +git commit -m "feat(schedule): read run records, timer state and per-run logs from the host journal + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 4: `ob status` reads the record + +**Files:** +- Modify: `internal/engine/schedule_status.go` (struct, batched command, parser), `internal/engine/status.go:168-187` (human lines) +- Test: `internal/engine/schedule_test.go` + +**Interfaces:** +- Produces new `StatusSchedule` fields: `NextRun string json:"next_run,omitempty"`, `LastOutcome string json:"last_outcome,omitempty"`, `LastDurationSeconds int json:"last_duration_s,omitempty"`, `LastAttempts int json:"last_attempts,omitempty"`, `ConsecutiveFailures int json:"consecutive_failures,omitempty"`, `Attempt int json:"attempt,omitempty"`, `JournalPersistent bool json:"journal_persistent"`. +- A record with outcome `failure` or `timeout` produces the issue `last run failed: (exit N)`; `skipped` and `success` produce none. With no record, today's systemd-based issue stays. + +- [ ] **Step 1: Write the failing test** + +```go +func TestScheduleStatusPrefersTheRunRecordOverSystemdResult(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: `@@journal +persistent +@@nightly:service +LoadState=loaded +ActiveState=inactive +Result=success +ExecMainStatus=75 +@@nightly:timer +LoadState=loaded +ActiveState=active +NextElapseUSecRealtime=Sat 2026-09-06 02:00:00 UTC +@@nightly:run +@@nightly:history +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","release":"r1","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:02Z","duration_s":1,"attempts":0,"exit_status":75,"outcome":"skipped","inputs":{}} +{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"timer","release":"r1","started_at":"2026-09-04T02:00:01Z","finished_at":"2026-09-04T02:05:02Z","duration_s":301,"attempts":3,"exit_status":1,"outcome":"failure","inputs":{}} +{"run":"c3d4e5f60718293a4b5c6d7e8f901234","job":"nightly","trigger":"timer","release":"r1","started_at":"2026-09-03T02:00:01Z","finished_at":"2026-09-03T02:01:02Z","duration_s":61,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}} +`}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + got := statuses[0] + if got.LastOutcome != "skipped" || got.NextRun != "Sat 2026-09-06 02:00:00 UTC" || !got.JournalPersistent { + t.Fatalf("record fields not surfaced: %#v", got) + } + if got.ConsecutiveFailures != 0 || got.Diverged { + t.Fatalf("a skip is not a failure: %#v", got) + } + if got.LastAttempts != 0 || got.LastDurationSeconds != 1 { + t.Fatalf("last run detail not surfaced: %#v", got) + } +} + +func TestScheduleStatusCountsConsecutiveFailuresFromRecords(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: `@@journal +volatile +@@nightly:service +LoadState=loaded +ActiveState=failed +Result=exit-code +ExecMainStatus=1 +@@nightly:timer +LoadState=loaded +ActiveState=active +@@nightly:run +@@nightly:history +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:02Z","duration_s":1,"attempts":2,"exit_status":1,"outcome":"failure","inputs":{}} +{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"timer","started_at":"2026-09-04T02:00:01Z","finished_at":"2026-09-04T02:05:02Z","duration_s":301,"attempts":2,"exit_status":null,"outcome":"timeout","inputs":{}} +{"run":"c3d4e5f60718293a4b5c6d7e8f901234","job":"nightly","trigger":"timer","started_at":"2026-09-03T02:00:01Z","finished_at":"2026-09-03T02:01:02Z","duration_s":61,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}} +`}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + got := statuses[0] + if got.ConsecutiveFailures != 2 || !got.Diverged || got.JournalPersistent { + t.Fatalf("failures not counted: %#v", got) + } + if !strings.Contains(strings.Join(got.Issues, "; "), "last run failed: failure (exit 1)") { + t.Fatalf("issue does not name the outcome: %#v", got.Issues) + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/engine -run 'TestScheduleStatusPrefersTheRunRecord|TestScheduleStatusCountsConsecutive'` +Expected: FAIL (unknown fields). + +- [ ] **Step 3: Implement** + +`internal/engine/schedule_status.go`: add the fields to `StatusSchedule`; add `history []ScheduleRunRecord` and `next string` to `scheduleUnitObservation`; extend the batched command. Replace the loop body building `commands` with: + +```go + commands := []string{ + "printf '%s\\n' '@@journal'", + "if [ -d /var/log/journal ]; then echo persistent; else echo volatile; fi", + } + for _, job := range jobs { + unit := e.names().ScheduledJobUnit(job.Name) + commands = append(commands, + "printf '%s\\n' "+q("@@"+job.Name+":service"), + "systemctl show "+q(unit+".service")+" --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus", + "printf '%s\\n' "+q("@@"+job.Name+":timer"), + "systemctl show "+q(unit+".timer")+" --no-pager --property=LoadState --property=ActiveState --property=NextElapseUSecRealtime", + "printf '%s\\n' "+q("@@"+job.Name+":run"), + "cat "+q(e.names().ScheduledJobRunState(job.Name))+" 2>/dev/null || true", + "printf '%s\\n' "+q("@@"+job.Name+":history"), + scheduleHistoryCommand(unit, 20), + ) + } +``` + +Parser: keep `values` for `key=value` kinds; for kind `history` collect raw lines; for marker `@@journal` (no colon) read the next non-empty line into `journalPersistent`. Replace the parse loop and `flush` with: + +```go + observed := map[string]map[string]scheduleUnitObservation{} + journalPersistent := true + name, kind := "", "" + values := map[string]string{} + var raw []string + flush := func() { + if name == "" || kind == "" { + return + } + if observed[name] == nil { + observed[name] = map[string]scheduleUnitObservation{} + } + exit, _ := strconv.Atoi(values["ExecMainStatus"]) + observed[name][kind] = scheduleUnitObservation{ + loadState: values["LoadState"], activeState: values["ActiveState"], + result: values["Result"], exitStatus: exit, + release: values["release"], startedAt: values["started_at"], attempt: values["attempt"], + next: values["NextElapseUSecRealtime"], + history: parseScheduleRunRecords(strings.Join(raw, "\n")), + } + values = map[string]string{} + raw = nil + } + for _, line := range strings.Split(res.Stdout, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "@@") { + flush() + marker := strings.TrimPrefix(line, "@@") + if marker == "journal" { + name, kind = "", "" + continue + } + name, kind, _ = strings.Cut(marker, ":") + continue + } + if name == "" { + if line == "volatile" { + journalPersistent = false + } + continue + } + if kind == "history" { + raw = append(raw, line) + continue + } + if key, value, ok := strings.Cut(line, "="); ok { + values[key] = value + } + } + flush() +``` + +Add `attempt`, `next string` and `history []ScheduleRunRecord` to `scheduleUnitObservation`. In the status assembly loop, after building `status`: + +```go + status.JournalPersistent = journalPersistent + status.NextRun = timer.next + if status.Running { + status.Attempt, _ = strconv.Atoi(run.attempt) + } + records := observed[job.Name]["history"].history + if len(records) > 0 { + last := records[0] + status.LastOutcome = last.Outcome + status.LastDurationSeconds = last.DurationSeconds + status.LastAttempts = last.Attempts + for _, record := range records { + if record.Outcome == "skipped" { + continue + } + if record.Outcome != "failure" && record.Outcome != "timeout" { + break + } + status.ConsecutiveFailures++ + } + } +``` + +Replace the systemd-result issue block with: + +```go + switch { + case status.LastOutcome == "failure" || status.LastOutcome == "timeout": + exit := "?" + if records[0].ExitStatus != nil { + exit = strconv.Itoa(*records[0].ExitStatus) + } + status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %s)", status.LastOutcome, exit)) + case status.LastOutcome == "": + if service.result != "" && service.result != "success" { + status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %d)", service.result, service.exitStatus)) + } + } +``` + +`internal/engine/status.go` human lines, replace the final `fmt.Fprintf(... "active; policy: ...")` with: + +```go + detail := fmt.Sprintf("active; policy: %s; timeout: %s", schedule.DeployLock, schedule.Timeout) + if schedule.NextRun != "" { + detail += "; next: " + schedule.NextRun + } + last := result + if schedule.LastOutcome != "" { + last = fmt.Sprintf("%s (%ds, %d attempt(s))", schedule.LastOutcome, schedule.LastDurationSeconds, schedule.LastAttempts) + } + detail += "; last: " + last + if !schedule.JournalPersistent { + detail += "; journal: volatile, history since boot only" + } + fmt.Fprintf(e.Opts.Out, "schedule %-11s %s\n", schedule.Name, detail) +``` + +and in the running branch append `fmt.Sprintf("; attempt: %d", schedule.Attempt)` when `schedule.Attempt > 0`. + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine ./cmd/ob` +Expected: PASS. The existing `TestScheduleStatusSurfacesTheLastSystemdFailure` still passes because it has no history block. + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine/schedule_status.go internal/engine/status.go internal/engine/schedule_test.go +git commit -m "feat(status): read scheduled-run outcomes, attempts and next elapse from the journal + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 5: CLI: `ob schedule list`, `ob schedule history`, `ob schedule logs` + +**Files:** +- Create: `cmd/ob/schedule.go` +- Modify: `cmd/ob/ops.go:69-88` (move `schedule` group construction into `addScheduleCommands(root, g)` in the new file), `cmd/ob/output.go:76-118` (matrix), `cmd/ob/output_test.go:492-530` (wantMatrix), `site/src/content/docs/reference/policies.mdx:98-102` +- Test: `cmd/ob/output_test.go`, `cmd/ob/docs_test.go` (regenerated `cli.mdx`) + +**Interfaces:** +- Consumes: `engine.ScheduleList`, `engine.ScheduleHistory`, `engine.ScheduleLogs`, `loadAllLenient`, `connect`, `newUI`, `writeFiniteSuccess`, `writeStructuredReadFailure`, `writeStructuredCommandFailure`, `isStructuredOutput`. +- Produces matrix rows: `"ob schedule list": {finite_envelope, JSON}`, `"ob schedule history": {finite_envelope, JSON}`, `"ob schedule logs": {operator_passthrough, JSON, NDJSON}`. + +- [ ] **Step 1: Update the closed-matrix test first** + +In `cmd/ob/output_test.go` `wantMatrix` add: + +```go + "ob schedule list": {Class: "finite_envelope", JSON: true}, + "ob schedule history": {Class: "finite_envelope", JSON: true}, + "ob schedule logs": {Class: "operator_passthrough", JSON: true, NDJSON: true}, +``` + +Run: `go test ./cmd/ob -run TestLeafOutputMatrixIsClosedAndHasNoAliases` +Expected: FAIL (matrix changed). + +- [ ] **Step 2: Implement `cmd/ob/schedule.go`** + +Move the existing `scheduleCmd`/`scheduleApplyCmd` construction from `ops.go` into `addScheduleCommands(root *cobra.Command, g *globalFlags)` and call it from where `root.AddCommand(scheduleCmd)` was. Then add: + +```go + var listCmd = &cobra.Command{ + Use: "list", + Short: "declared scheduled jobs with timer state and next elapse", + Long: "List every job that declares a schedule beside what the host's timer says: whether it is active, when it fires next, and when it last fired. Reads only.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, p, err := loadAllLenient(cmd.Context(), g) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + defer cleanup() + jobs, err := e.ScheduleList(cmd.Context()) + if err != nil { + return writeStructuredCommandFailure(cmd, g, "schedule_list_failed", "scheduled jobs could not be listed", err) + } + if isStructuredOutput(g) { + return writeFiniteSuccess(cmd, g, map[string]any{"jobs": jobs}) + } + if len(jobs) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "no scheduled jobs declared") + return nil + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "JOB\tCRON\tTZ\tTIMER\tNEXT\tLAST TRIGGER\tPOLICY\tTIMEOUT") + for _, j := range jobs { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", j.Name, j.Cron, j.Timezone, orDash(j.TimerState), orDash(j.NextRun), orDash(j.LastTrigger), j.DeployLock, j.Timeout) + } + return w.Flush() + }, + } +``` + +History: + +```go + var historyCount int + historyCmd := &cobra.Command{ + Use: "history ", + Short: "run records of one scheduled job, newest first", + Long: "Read the run records the host wrote for one scheduled job. Each record is one activation: run id, trigger, release, start and end, attempts, exit status, outcome and, for a manual run, its inputs.\n\nRecords live in the host journal under the job's unit with syslog identifier ob-run; retention is the journal's. Reads only.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, p, err := loadAllLenient(cmd.Context(), g) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + defer cleanup() + records, err := e.ScheduleHistory(cmd.Context(), args[0], historyCount) + if err != nil { + return writeStructuredCommandFailure(cmd, g, "schedule_history_failed", "run history could not be read", err) + } + if isStructuredOutput(g) { + return writeFiniteSuccess(cmd, g, map[string]any{"job": args[0], "runs": records}) + } + if len(records) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "no recorded runs for %s\n", args[0]) + return nil + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "STARTED\tOUTCOME\tDURATION\tATTEMPTS\tEXIT\tTRIGGER\tRELEASE\tRUN") + for _, r := range records { + exit := "-" + if r.ExitStatus != nil { + exit = strconv.Itoa(*r.ExitStatus) + } + fmt.Fprintf(w, "%s\t%s\t%ds\t%d\t%s\t%s\t%s\t%s\n", r.StartedAt, r.Outcome, r.DurationSeconds, r.Attempts, exit, r.Trigger, orDash(r.Release), r.Run) + } + return w.Flush() + }, + } + historyCmd.Flags().IntVarP(&historyCount, "count", "n", 20, "number of newest runs to show") +``` + +Logs: mirror `ob logs` in `cmd/ob/ops.go:229-300` exactly (JSON buffers `stdout`/`stderr` into `{"job","run","stdout","stderr","passthrough_unredacted":true}`; NDJSON uses the same streaming writer `ob logs` uses; human streams to the terminal). Flags: `--run ` and `-n/--tail` (default 200). Failure code `schedule_logs_failed`, message `run logs could not be read`. + +`orDash`: + +```go +func orDash(s string) string { + if strings.TrimSpace(s) == "" { + return "-" + } + return s +} +``` + +Add the three matrix rows to `cliOutputMatrix` in `output.go`. In `policies.mdx` add `` `ob schedule history` `` and `` `ob schedule list` `` to the Finite envelope row (alphabetical position after `ob preview`) and `` `ob schedule logs` `` to the `Operator passthrough | finite only | yes` row beside `ob logs`. + +- [ ] **Step 3: Regenerate docs and run tests** + +Run: `just build && just docs-generate && go test ./cmd/ob ./internal/... && just docs-generate-check` +Expected: PASS; `cli.mdx` gains the three commands. + +- [ ] **Step 4: Commit** + +```bash +git add cmd/ob/schedule.go cmd/ob/ops.go cmd/ob/output.go cmd/ob/output_test.go site/src/content/docs/reference/policies.mdx site/src/content/docs/reference/cli.mdx +git commit -m "feat(cli): ob schedule list, history and logs read the host's run records + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +## Slice 2: Retry and notify + +### Task 6: Schema, validation and defaults for `retry` and `notify` + +**Files:** +- Modify: `internal/app/types.go:268-274` (`JobSchedule`), `internal/app/constraints.go:173` (add `eScheduleNotify`), `internal/app/validate.go:573-584` (`validateJobSchedule`), `internal/app/schedule.go:26-64` (`ScheduledJob`, `ScheduledJobs`) +- Test: `internal/app/schedule_test.go` + +**Interfaces:** +- Produces: + - `type JobRetry struct { Attempts int; Backoff string; MaxBackoff string }` JSON `attempts`, `backoff`, `max_backoff`, with `description` and `default` tags. + - `JobSchedule.Retry *JobRetry json:"retry,omitempty"`, `JobSchedule.Notify []string json:"notify,omitempty"`. + - `ScheduledJob` gains `RetryAttempts int`, `RetryBackoff time.Duration`, `RetryMaxBackoff time.Duration`, `Notify []string`, resolved with defaults in `ScheduledJobs()`. + - `func scheduleRetryWorstCase(attempts int, backoff, max time.Duration) time.Duration` = sum over the attempts-1 sleeps of `min(backoff·2^i, max)`. + +- [ ] **Step 1: Write the failing tests** + +Append to `internal/app/schedule_test.go`: + +```go +func TestScheduledJobRetryAndNotifyResolveWithDefaults(t *testing.T) { + spec, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + plain: + role: job + image: x:1 + data_effect: none + schedule: {cron: "0 3 * * *"} + retrying: + role: job + image: x:1 + data_effect: none + schedule: + cron: "0 * * * *" + timeout: 45m + retry: {attempts: 3, backoff: 30s, max_backoff: 10m} + notify: [failure, timeout, skipped] +`), "ob.yml") + if err != nil { + t.Fatal(err) + } + jobs, err := spec.ScheduledJobs() + if err != nil { + t.Fatal(err) + } + byName := map[string]ScheduledJob{} + for _, job := range jobs { + byName[job.Name] = job + } + plain := byName["plain"] + if plain.RetryAttempts != 1 || plain.RetryBackoff != 30*time.Second || plain.RetryMaxBackoff != 10*time.Minute || + strings.Join(plain.Notify, ",") != "failure,timeout" { + t.Fatalf("defaults did not resolve: %#v", plain) + } + retrying := byName["retrying"] + if retrying.RetryAttempts != 3 || strings.Join(retrying.Notify, ",") != "failure,timeout,skipped" { + t.Fatalf("declared retry did not resolve: %#v", retrying) + } +} + +func TestScheduledJobRetryIsBoundedByTheTimeout(t *testing.T) { + for name, tc := range map[string]struct { + schedule string + code string + }{ + "too many attempts": {`{cron: "0 * * * *", retry: {attempts: 11}}`, "project_invalid"}, + "zero attempts": {`{cron: "0 * * * *", retry: {attempts: 0}}`, "project_invalid"}, + "backoff over max": {`{cron: "0 * * * *", retry: {attempts: 2, backoff: 20m, max_backoff: 10m}}`, "project_invalid"}, + "backoff exceeds timeout": {`{cron: "0 * * * *", timeout: 5m, retry: {attempts: 3, backoff: 2m, max_backoff: 30m}}`, "project_invalid"}, + "unknown notify": {`{cron: "0 * * * *", notify: [warning]}`, "project_invalid"}, + } { + t.Run(name, func(t *testing.T) { + _, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + j: {role: job, image: x:1, data_effect: none, schedule: `+tc.schedule+`} +`), "ob.yml") + var e *Error + if !errors.As(err, &e) || e.Code != tc.code { + t.Fatalf("err = %v, want code %s", err, tc.code) + } + }) + } + if got := scheduleRetryWorstCase(3, 2*time.Minute, 30*time.Minute); got != 6*time.Minute { + t.Fatalf("worst case = %s, want 6m", got) + } + if got := scheduleRetryWorstCase(4, 30*time.Second, time.Minute); got != 150*time.Second { + t.Fatalf("capped worst case = %s, want 2m30s", got) + } +} +``` + +Add `"errors"`, `"strings"`, `"time"` to that test file's imports if absent. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/app -run 'TestScheduledJobRetry'` +Expected: FAIL to compile. + +- [ ] **Step 3: Implement** + +`types.go`: + +```go +type JobSchedule struct { + Cron string `json:"cron" ...unchanged...` + Timezone string ... + Timeout string ... + CatchUp bool ... + DeployLock string ... + Retry *JobRetry `json:"retry,omitempty" description:"Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run."` + Notify []string `json:"notify,omitempty" description:"Run outcomes that send the configured notifications: success, failure, timeout, skipped." default:"failure, timeout"` +} + +type JobRetry struct { + Attempts int `json:"attempts,omitempty" description:"Total attempts including the first, 1 to 10." default:"1" example:"3"` + Backoff string `json:"backoff,omitempty" description:"Sleep before the second attempt; it doubles after each failure." default:"30s" example:"1m"` + MaxBackoff string `json:"max_backoff,omitempty" description:"Upper bound for the doubling sleep." default:"10m" example:"30m"` +} +``` + +`constraints.go` beside `eNotifyEvent`: `eScheduleNotify = []string{"success", "failure", "timeout", "skipped"}`. + +`schedule.go`: add constants and resolution: + +```go +const ( + defaultRetryAttempts = 1 + defaultRetryBackoff = 30 * time.Second + defaultRetryMaxBackoff = 10 * time.Minute +) + +var defaultScheduleNotify = []string{"failure", "timeout"} + +// scheduleRetryWorstCase is the longest a run can spend asleep between +// attempts. Validation keeps it under the timeout, so the last attempt can +// always start. +func scheduleRetryWorstCase(attempts int, backoff, max time.Duration) time.Duration { + var total time.Duration + sleep := backoff + for i := 1; i < attempts; i++ { + if sleep > max { + sleep = max + } + total += sleep + sleep *= 2 + } + return total +} + +func (s *JobSchedule) retryPolicy() (int, time.Duration, time.Duration) { + attempts, backoff, max := defaultRetryAttempts, defaultRetryBackoff, defaultRetryMaxBackoff + if s.Retry != nil { + if s.Retry.Attempts > 0 { + attempts = s.Retry.Attempts + } + if d, ok := ParseDuration(s.Retry.Backoff); ok && s.Retry.Backoff != "" { + backoff = d + } + if d, ok := ParseDuration(s.Retry.MaxBackoff); ok && s.Retry.MaxBackoff != "" { + max = d + } + } + return attempts, backoff, max +} +``` + +Extend `ScheduledJob` with `RetryAttempts int`, `RetryBackoff, RetryMaxBackoff time.Duration`, `Notify []string`; in `ScheduledJobs()` fill them: + +```go + attempts, backoff, max := w.Schedule.retryPolicy() + notify := w.Schedule.Notify + if len(notify) == 0 { + notify = append([]string(nil), defaultScheduleNotify...) + } + out = append(out, ScheduledJob{ + Name: name, Cron: w.Schedule.Cron, Timezone: tz, Calendar: cal, + Timeout: w.Schedule.Timeout, CatchUp: w.Schedule.CatchUp, DeployLock: deployLock, + RetryAttempts: attempts, RetryBackoff: backoff, RetryMaxBackoff: max, Notify: notify, + }) +``` + +`validate.go` `validateJobSchedule`, after the timeout check: + +```go + if s.Retry != nil { + if s.Retry.Attempts < 0 || s.Retry.Attempts > 10 { + return errf("project_invalid", path+".retry.attempts", "", "attempts must be between 1 and 10, got %d", s.Retry.Attempts) + } + if raw, err := yamlZeroAttempts(s); err == nil && raw { + return errf("project_invalid", path+".retry.attempts", "", "attempts must be between 1 and 10, got 0") + } + if err := gDur.checkOptional(path+".retry.backoff", s.Retry.Backoff); err != nil { + return err + } + if err := gDur.checkOptional(path+".retry.max_backoff", s.Retry.MaxBackoff); err != nil { + return err + } + attempts, backoff, max := s.retryPolicy() + if backoff > max { + return errf("project_invalid", path+".retry.backoff", "", "backoff %s exceeds max_backoff %s", s.Retry.Backoff, s.Retry.MaxBackoff) + } + timeout := time.Hour + if s.Timeout != "" { + if d, ok := ParseDuration(s.Timeout); ok { + timeout = d + } + } + if worst := scheduleRetryWorstCase(attempts, backoff, max); worst >= timeout { + return errf("project_invalid", path+".retry", "", + "worst-case backoff %s is not smaller than timeout %s; later attempts could never start", worst, timeout) + } + } + for i, outcome := range s.Notify { + if err := checkEnum(indexed(path+".notify", i), outcome, eScheduleNotify); err != nil { + return err + } + } +``` + +`attempts: 0` cannot be told from "absent" after decoding into `int`. Make `Attempts` a `*int`? Simpler and honest: change `JobRetry.Attempts` to `int` and treat `0` as "not set" only when the whole `retry` block is absent; when `retry:` is present with `attempts: 0`, the raw YAML check `applyDefaults` already receives `raw map[string]any`. Rather than a raw lookup, declare `Attempts *int` with `default:"1"` and validate `*Attempts` in 1..10 when non-nil; `retryPolicy` uses `*Attempts` when non-nil. Delete the `yamlZeroAttempts` line above and implement with the pointer. `schemaTagValue` handles `*int` via `deref` for the default tag; verify with `TestPublishedSchemaDocumentsEveryPublicField`. + +- [ ] **Step 4: Run tests and regenerate schema** + +Run: `go test ./internal/app && go run ./cmd/ob schema --out docs/onebox.run-v1.schema.json && go test ./internal/app ./cmd/ob-docgen` +Expected: PASS. `TestContractDidNotMove` still passes: no existing case declares `retry` or `notify`, and neither field renders into Compose. + +- [ ] **Step 5: Commit** + +```bash +git add internal/app docs/onebox.run-v1.schema.json +git commit -m "feat(schema): schedule.retry and schedule.notify with validation against the timeout + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 7: Runner retry loop and per-outcome notifications + +**Files:** +- Modify: `internal/engine/schedule.go` (`scheduleRunnerScript`, `pinnedScheduleRunnerScript`, `scheduleFailureNotifier`) +- Test: `internal/engine/schedule_test.go` + +**Interfaces:** +- Produces: `func scheduleAttemptLoop(job app.ScheduledJob, compose string) []string`; notifier renders a success body and a failure body per notification and selects by outcome; `scheduleFailureNotifier(job string)` becomes `scheduleNotifier(job app.ScheduledJob)`. + +- [ ] **Step 1: Write the failing tests** + +```go +func TestScheduledJobRunnerRetriesWithCappedDoublingBackoff(t *testing.T) { + job := app.ScheduledJob{Name: "nightly", Timeout: "45m", DeployLock: "exclusive", + RetryAttempts: 3, RetryBackoff: 30 * time.Second, RetryMaxBackoff: 10 * time.Minute} + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil) + for _, want := range []string{ + "max_attempts=3", "backoff=30", "max_backoff=600", + "attempt=1", "while :; do", "write_state \"$attempt\"", + "status=0", "|| status=$?", "[ \"$status\" -eq 0 ] && exit 0", + "if [ \"$attempt\" -ge \"$max_attempts\" ]; then exit \"$status\"; fi", + "sleep \"$backoff\"", "backoff=$((backoff * 2))", "attempt=$((attempt + 1))", + } { + if !strings.Contains(runner, want) { + t.Errorf("runner is missing %q:\n%s", want, runner) + } + } + single := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil) + if strings.Contains(single, "while :; do") { + t.Errorf("a single-attempt job must not carry a retry loop:\n%s", single) + } + for _, script := range []string{runner, single} { + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(script) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("runner is not valid POSIX shell: %v: %s\n%s", err, output, script) + } + } +} + +func TestScheduledJobNotifierSelectsBodiesByOutcome(t *testing.T) { + cfg := testConfig() + cfg.Notifications = map[string]app.Notification{ + "ops": {Webhook: "https://hooks.example.com/ops", On: []string{"success", "failure"}, Format: "json"}, + } + f := &transport.Fake{TargetName: "root@example.internal"} + e := New(cfg, testProject(t), f, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + script, err := e.scheduleNotifier(app.ScheduledJob{Name: "nightly", Notify: []string{"success", "failure", "skipped"}}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `case " success failure skipped " in *" $outcome "*) ;; *) exit 0 ;; esac`, + `"status":"ok"`, + `"status":"fail"`, + `if [ "$outcome" = success ]; then`, + `"$detail"`, + `detail="run ${INVOCATION_ID:-?}: $outcome after $attempt attempt(s) in ${duration}s (exit $status)"`, + } { + if !strings.Contains(script, want) { + t.Errorf("notifier is missing %q:\n%s", want, script) + } + } +} +``` + +Update `TestScheduledJobFailureNotifierUsesConfiguredWebhooks` to call `e.scheduleNotifier(app.ScheduledJob{Name: "nightly", Notify: []string{"failure", "timeout"}})` and keep its expectations; the `success-only` webhook is still absent because its `On` excludes failure and the job's `Notify` excludes success. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/engine -run 'TestScheduledJobRunnerRetries|TestScheduledJobNotifierSelects'` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +`scheduleAttemptLoop`: + +```go +// scheduleAttemptLoop runs the container until it exits 0 or the attempts are +// spent. Backoff doubles and is capped; every sleep happens under the locks the +// run already holds, which is why validation keeps the sum under the timeout. +func scheduleAttemptLoop(job app.ScheduledJob, compose string) []string { + if job.RetryAttempts <= 1 { + return []string{"write_state 1", compose} + } + return []string{ + fmt.Sprintf("max_attempts=%d", job.RetryAttempts), + fmt.Sprintf("backoff=%d", int(math.Ceil(job.RetryBackoff.Seconds()))), + fmt.Sprintf("max_backoff=%d", int(math.Ceil(job.RetryMaxBackoff.Seconds()))), + "attempt=1", + "while :; do", + " write_state \"$attempt\"", + " status=0", + " " + compose + " || status=$?", + " [ \"$status\" -eq 0 ] && exit 0", + " if [ \"$attempt\" -ge \"$max_attempts\" ]; then exit \"$status\"; fi", + " echo \"onebox: attempt $attempt of $max_attempts exited $status; retrying in ${backoff}s\" >&2", + " sleep \"$backoff\"", + " backoff=$((backoff * 2))", + " [ \"$backoff\" -gt \"$max_backoff\" ] && backoff=$max_backoff", + " attempt=$((attempt + 1))", + "done", + } +} +``` + +Both runners: replace `lines = append(lines, "write_state 1", compose, "")` with `lines = append(lines, scheduleAttemptLoop(job, compose)...); lines = append(lines, "")` (pinned keeps `"/usr/bin/flock --unlock 8"` before the loop). `pinnedScheduleRunnerScript` takes `job app.ScheduledJob` instead of `job string`; update its two callers. + +Notifier: rename to `scheduleNotifier(job app.ScheduledJob)`; after the record lines: + +```go + lines = append(lines, + `detail="run ${INVOCATION_ID:-?}: $outcome after $attempt attempt(s) in ${duration}s (exit $status)"`, + `case " `+strings.Join(job.Notify, " ")+` " in *" $outcome "*) ;; *) exit 0 ;; esac`, + ) +``` + +For each notification prepare two payloads: `Status: "ok", Error: ""` and `Status: "fail", Error: scheduleNotificationDetail` where `const scheduleNotificationDetail = "__ONEBOX_SCHEDULE_DETAIL__"`; substitute both `scheduleNotificationTimestamp` with `"$ts"` and `scheduleNotificationDetail` with `"$detail"` (same `strings.Cut` technique, applied twice through a small helper `shellBody(body string) string`). Emit: + +``` +ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ') +if [ "$outcome" = success ]; then + +else + +fi +wait || true +exit 0 +``` + +Both branches may be empty; render `:` in an empty branch so the shell stays valid. The verb stays `"scheduled job " + job.Name`; the `Text` line for text-format webhooks carries the detail through the same placeholder. + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine ./cmd/...` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine +git commit -m "feat(schedule): bounded retry in the runner and per-outcome notifications + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +## Slice 3: Inputs and manual runs + +### Task 8: Schema and validation for `inputs`; defaults rendered into Compose + +**Files:** +- Modify: `internal/app/types.go:136-174` (`Workload.Inputs`), `internal/app/validate.go:436-458`, `internal/app/generate.go:347` (environment), `internal/app/schedule.go` (`ScheduledJob.Inputs`), `internal/app/constraints.go`, `internal/app/names.go:242` +- Test: `internal/app/schedule_test.go`, `internal/app/generate_test.go` (or the nearest existing render test file) + +**Interfaces:** +- Produces: + - `type JobInput struct { Enum []string; Pattern string; Default string; Description string }` JSON `enum, pattern, default, description`. + - `Workload.Inputs map[string]JobInput json:"inputs,omitempty"`. + - `ScheduledJob.Inputs map[string]JobInput`. + - `func ValidateJobInputValues(w Workload, values map[string]string) error` (used by the engine for overrides): unknown name, pattern/enum mismatch, charset, length. + - `var gInputName = grammar{"input name", ^[A-Z][A-Z0-9_]*$}`, `const ReservedInputPrefix = "ONEBOX_"`, `func InputValueAllowed(v string) bool` (≤256 bytes, no `"`, `\`, control chars). + - `func (n Names) ScheduledJobRunInputs(job string) string` = `/schedule/.inputs`. + +- [ ] **Step 1: Write the failing tests** + +```go +func TestJobInputsValidateNamesConstraintsAndDefaults(t *testing.T) { + base := `api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + sync: + role: job + image: x:1 + data_effect: %s + env: {MODE: fast} + %s + inputs: + %s +` + load := func(effect, schedule, inputs string) error { + _, err := LoadBytes([]byte(fmt.Sprintf(base, effect, schedule, inputs)), "ob.yml") + return err + } + good := "SOURCE: {enum: [catalog, prices], default: catalog, description: Which upstream.}" + if err := load("none", `schedule: {cron: "0 * * * *"}`, good); err != nil { + t.Fatalf("valid inputs refused: %v", err) + } + for name, tc := range map[string]struct{ effect, schedule, inputs string }{ + "no schedule": {"none", "", good}, + "destructive job": {"destructive", `schedule: {cron: "0 * * * *"}`, good}, + "lowercase name": {"none", `schedule: {cron: "0 * * * *"}`, "source: {enum: [a], default: a}"}, + "reserved prefix": {"none", `schedule: {cron: "0 * * * *"}`, "ONEBOX_X: {enum: [a], default: a}"}, + "collides with env": {"none", `schedule: {cron: "0 * * * *"}`, "MODE: {enum: [a], default: a}"}, + "enum and pattern": {"none", `schedule: {cron: "0 * * * *"}`, "S: {enum: [a], pattern: '^a$', default: a}"}, + "neither": {"none", `schedule: {cron: "0 * * * *"}`, "S: {default: a}"}, + "default off enum": {"none", `schedule: {cron: "0 * * * *"}`, "S: {enum: [a], default: b}"}, + "default off pattern": {"none", `schedule: {cron: "0 * * * *"}`, "S: {pattern: '^[0-9]+$', default: x}"}, + "quote in default": {"none", `schedule: {cron: "0 * * * *"}`, `S: {pattern: '.*', default: 'a"b'}`}, + "bad regex": {"none", `schedule: {cron: "0 * * * *"}`, "S: {pattern: '(', default: a}"}, + } { + t.Run(name, func(t *testing.T) { + var e *Error + if err := load(tc.effect, tc.schedule, tc.inputs); !errors.As(err, &e) || e.Code != "project_invalid" { + t.Fatalf("err = %v, want project_invalid", err) + } + }) + } +} + +func TestValidateJobInputValuesChecksOverrides(t *testing.T) { + w := Workload{Role: RoleJob, Inputs: map[string]JobInput{ + "SOURCE": {Enum: []string{"catalog", "prices"}, Default: "catalog"}, + "SINCE": {Pattern: `^([0-9]{4}-[0-9]{2}-[0-9]{2})?$`, Default: ""}, + }} + if err := ValidateJobInputValues(w, map[string]string{"SOURCE": "prices", "SINCE": "2026-09-01"}); err != nil { + t.Fatal(err) + } + for name, values := range map[string]map[string]string{ + "unknown": {"OTHER": "x"}, + "off enum": {"SOURCE": "reviews"}, + "off pattern": {"SINCE": "yesterday"}, + "backslash": {"SINCE": `2026\-09-01`}, + "newline": {"SOURCE": "prices\n"}, + } { + if err := ValidateJobInputValues(w, values); err == nil { + t.Errorf("%s was accepted", name) + } + } + if InputValueAllowed(strings.Repeat("a", 257)) { + t.Error("an oversized value was allowed") + } +} + +func TestScheduledJobInputDefaultsRenderIntoTheComposeEnvironment(t *testing.T) { + spec, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + sync: + role: job + image: x:1 + data_effect: none + env: {MODE: fast} + schedule: {cron: "0 * * * *"} + inputs: + SOURCE: {enum: [catalog, prices], default: catalog} +`), "ob.yml") + if err != nil { + t.Fatal(err) + } + rendered := renderForTest(t, spec) // use the same helper the existing generate tests use to render Compose YAML + for _, want := range []string{"SOURCE: catalog", "MODE: fast"} { + if !strings.Contains(rendered, want) { + t.Fatalf("rendered runtime is missing %q:\n%s", want, rendered) + } + } + jobs, _ := spec.ScheduledJobs() + if jobs[0].Inputs["SOURCE"].Default != "catalog" { + t.Fatalf("scheduled job did not carry its inputs: %#v", jobs[0]) + } +} +``` + +Before writing the render test, locate the render helper with `rtk rg -n 'func render.*\(t \*testing.T' internal/app/*_test.go` and use its real name in place of `renderForTest`. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/app -run 'TestJobInputs|TestValidateJobInputValues|TestScheduledJobInputDefaults'` +Expected: FAIL to compile. + +- [ ] **Step 3: Implement** + +`types.go`, in `Workload` under `// Job only.`: + +```go + Inputs map[string]JobInput `json:"inputs,omitempty" description:"Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them."` +``` + +and + +```go +type JobInput struct { + Enum []string `json:"enum,omitempty" description:"Accepted values." example:"catalog"` + Pattern string `json:"pattern,omitempty" description:"Regular expression the whole value must match." example:"^[0-9]{4}-[0-9]{2}-[0-9]{2}$"` + Default string `json:"default" description:"Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint."` + Description string `json:"description,omitempty" description:"What the input controls."` +} +``` + +`constraints.go`: + +```go + gInputName = grammar{"input name", regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`), + "upper-case letters, digits and underscores, starting with a letter"} +``` + +(match the existing `grammar` literal shape; the third field is the hint string if the struct has one). + +`schedule.go` (app): + +```go +// ReservedInputPrefix is Onebox's environment namespace; declared inputs may +// not enter it, and the manual-run metadata line lives there. +const ReservedInputPrefix = "ONEBOX_" + +const maxInputValueBytes = 256 + +// InputValueAllowed is the charset that lets the runner pass a value through +// to the container and into the run record without escaping anything. +func InputValueAllowed(v string) bool { + if len(v) > maxInputValueBytes { + return false + } + for _, r := range v { + if r == '"' || r == '\\' || r < 0x20 || r == 0x7f { + return false + } + } + return true +} + +func (in JobInput) accepts(v string) bool { + if !InputValueAllowed(v) { + return false + } + if len(in.Enum) > 0 { + for _, allowed := range in.Enum { + if v == allowed { + return true + } + } + return false + } + re, err := regexp.Compile(in.Pattern) + if err != nil { + return false + } + return re.MatchString(v) && re.FindStringIndex(v) != nil && wholeMatch(re, v) +} + +func wholeMatch(re *regexp.Regexp, v string) bool { + loc := re.FindStringIndex(v) + return loc != nil && loc[0] == 0 && loc[1] == len(v) +} + +// ValidateJobInputValues checks operator-supplied overrides against the +// declaration. Defaults were checked at load; this is the other half. +func ValidateJobInputValues(w Workload, values map[string]string) error { + for _, name := range sortedKeys(values) { + in, ok := w.Inputs[name] + if !ok { + return fmt.Errorf("input %s is not declared", name) + } + if !in.accepts(values[name]) { + return fmt.Errorf("input %s: %q is not an accepted value", name, values[name]) + } + } + return nil +} +``` + +(`sortedKeys` is generic over map value type in this package? Check its signature at `internal/app/schedule.go` usage; if it is `map[string]Workload`-specific, add a local `sortedStringKeys(map[string]string)`.) In `ScheduledJobs()` add `Inputs: w.Inputs` to the literal and `Inputs map[string]JobInput` to `ScheduledJob`. + +`validate.go` inside `if w.IsJob() {` after the pinned block: + +```go + if len(w.Inputs) > 0 { + if w.Schedule == nil { + return errf("project_invalid", path+".inputs", "", "inputs belong to a scheduled job; declare schedule or remove inputs") + } + if w.DataEffect != DataEffectNone { + return errf("project_invalid", path+".inputs", "", + "inputs require data_effect %q; a %q job keeps the sealed plan of ob job run for operator-initiated runs", DataEffectNone, w.DataEffect) + } + for _, name := range sortedKeys(w.Inputs) { + ip := path + ".inputs." + name + if err := gInputName.check(ip, name); err != nil { + return err + } + if strings.HasPrefix(name, ReservedInputPrefix) { + return errf("project_invalid", ip, "", "%s is in Onebox's environment namespace", ReservedInputPrefix) + } + if _, clash := w.Env[name]; clash { + return errf("project_invalid", ip, "", "input %s collides with an env key of the same name", name) + } + in := w.Inputs[name] + switch { + case len(in.Enum) > 0 && in.Pattern != "": + return errf("project_invalid", ip, "", "declare exactly one of enum or pattern") + case len(in.Enum) == 0 && in.Pattern == "": + return errf("project_invalid", ip, "", "declare exactly one of enum or pattern") + } + if in.Pattern != "" { + if _, err := regexp.Compile(in.Pattern); err != nil { + return errf("project_invalid", ip+".pattern", "", "%v", err) + } + } + for i, v := range in.Enum { + if !InputValueAllowed(v) { + return errf("project_invalid", indexed(ip+".enum", i), "", "enum values may not contain quotes, backslashes or control characters and are at most %d bytes", maxInputValueBytes) + } + } + if !in.accepts(in.Default) { + return errf("project_invalid", ip+".default", "", "default %q does not satisfy the input's own constraint", in.Default) + } + } + } +``` + +Also in the `else if` branch for non-jobs add `|| len(w.Inputs) > 0` to the condition and mention `inputs` in the message. + +`generate.go:347`: + +```go + env := stringMap(w.Env) + if len(w.Inputs) > 0 { + if env == nil { + env = map[string]any{} + } + for name, in := range w.Inputs { + env[name] = in.Default + } + } + if len(env) > 0 { + svc["environment"] = env + } +``` + +`names.go`: + +```go +// ScheduledJobRunInputs is the one-shot file ob schedule run leaves for the +// next manual activation. The runner consumes and deletes it. +func (n Names) ScheduledJobRunInputs(job string) string { + return path.Join(n.AppDir(), "schedule", job+".inputs") +} +``` + +- [ ] **Step 4: Run tests, regenerate schema** + +Run: `go test ./internal/app && go run ./cmd/ob schema --out docs/onebox.run-v1.schema.json && go test ./internal/app ./cmd/ob-docgen` +Expected: PASS. If `TestContractDidNotMove` fails, no existing case uses `inputs`, so investigate before ever running `-update`. + +- [ ] **Step 5: Commit** + +```bash +git add internal/app docs/onebox.run-v1.schema.json +git commit -m "feat(schema): declared inputs for scheduled jobs, rendered as environment defaults + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 9: Runner consumes manual inputs; host systemd floor + +**Files:** +- Modify: `internal/engine/schedule.go` (both runners, `SyncSchedules`) +- Test: `internal/engine/schedule_test.go` + +**Interfaces:** +- Produces: `func scheduleInputsLines(inputsPath string) []string` inserted after the preamble in both runners; the compose command gains `"$@"` after `run --rm --no-deps`; `SyncSchedules` checks `systemctl --version` ≥ 252 when any job has inputs; error text: `job %s declares inputs, which need systemd 252 or newer on the host for $TRIGGER_UNIT; the host runs %s`. + +- [ ] **Step 1: Write the failing tests** + +```go +func TestScheduledJobRunnerConsumesManualInputsWithoutShellInterpolation(t *testing.T) { + job := app.ScheduledJob{Name: "sync", Timeout: "45m", DeployLock: "pinned", RetryAttempts: 1, + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog"}, Default: "catalog"}}} + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil) + for _, want := range []string{ + "inputs_file='/var/lib/ob/sample/schedule/sync.inputs'", + `if [ -z "${TRIGGER_UNIT:-}" ] && [ -f "$inputs_file" ]; then`, + `while IFS= read -r line || [ -n "$line" ]; do`, + `ONEBOX_OPERATION=*) operation=${line#ONEBOX_OPERATION=} ;;`, + `[A-Z]*=*) set -- "$@" -e "$line"`, + `rm -f "$inputs_file"`, + `run --rm --no-deps "$@" --name 'sample-sync-1' 'sync'`, + } { + if !strings.Contains(runner, want) { + t.Errorf("runner is missing %q:\n%s", want, runner) + } + } + if strings.Contains(runner, ". \"$inputs_file\"") || strings.Contains(runner, "eval") { + t.Fatalf("runner evaluates the inputs file as shell:\n%s", runner) + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(runner) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("runner is not valid POSIX shell: %v: %s\n%s", err, output, runner) + } + // The consume block precedes the locks so a skipped manual run cannot + // leave its inputs for the next timer firing. + if strings.Index(runner, "inputs_file=") > strings.Index(runner, "flock --exclusive --nonblock --conflict-exit-code 75 9") { + t.Fatalf("inputs are consumed after the lock:\n%s", runner) + } +} + +func TestSyncSchedulesRequiresSystemd252ForInputs(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"a"}, Default: "a"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "list-unit-files"): + return transport.Result{}, true + case strings.Contains(cmd, "systemd-analyze calendar"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl --version"): + return transport.Result{Stdout: "systemd 249 (249.11-0ubuntu3)\n"}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + err := e.SyncSchedules(context.Background()) + if err == nil || !strings.Contains(err.Error(), "systemd 252") { + t.Fatalf("old systemd was accepted for a job with inputs: %v", err) + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/engine -run 'TestScheduledJobRunnerConsumesManualInputs|TestSyncSchedulesRequiresSystemd252'` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +```go +// scheduleInputsLines consumes the one-shot inputs file on a manual +// activation. Values reach the container as -e arguments, never as shell +// text, and the file is gone before any lock is taken so a skipped manual run +// cannot hand its inputs to the next timer firing. +func scheduleInputsLines(inputsPath string) []string { + return []string{ + "inputs_file=" + q(inputsPath), + "if [ -z \"${TRIGGER_UNIT:-}\" ] && [ -f \"$inputs_file\" ]; then", + " while IFS= read -r line || [ -n \"$line\" ]; do", + " case \"$line\" in", + " ONEBOX_OPERATION=*) operation=${line#ONEBOX_OPERATION=} ;;", + " [A-Z]*=*) set -- \"$@\" -e \"$line\"; key=${line%%=*}; value=${line#*=}; inputs_json=\"${inputs_json:+$inputs_json,}\\\"$key\\\":\\\"$value\\\"\" ;;", + " esac", + " done <\"$inputs_file\"", + " rm -f \"$inputs_file\"", + "fi", + } +} +``` + +Order inside both runners: shebang, `set -eu`, `install -d`, then `operation=''`, `inputs_json=''`, then `scheduleInputsLines(...)`, then the flocks and the rest; move the two variable initialisations out of `scheduleRunPreamble` (keep the preamble's `state`, `tmp`, `started_*`, `trigger`, `write_state`). The compose command string in both runners becomes: + +```go + compose := "/usr/bin/docker compose -p " + q(application) + " --project-directory " + projectDir + + " -f " + projectDir + "/" + q("compose.yaml") + scheduleRuntimeEnvArgs(projectDir, runtimeEnvFiles) + + " run --rm --no-deps \"$@\" --name " + q(container) + " " + q(job.Name) +``` + +In `SyncSchedules`, before the per-job loop: + +```go + if needsTriggerUnit(jobs) { + res, err := e.T.Run(ctx, "systemctl --version 2>/dev/null | head -1") + if err != nil { + return err + } + if version, ok := systemdVersion(res.Stdout); !ok || version < 252 { + return fmt.Errorf("a job declares inputs, which need systemd 252 or newer on the host for $TRIGGER_UNIT; the host reports %q", strings.TrimSpace(res.Stdout)) + } + } +``` + +with + +```go +func needsTriggerUnit(jobs []app.ScheduledJob) bool { + for _, job := range jobs { + if len(job.Inputs) > 0 { + return true + } + } + return false +} + +// systemdVersion reads the leading number from `systemd 255 (255.4-1ubuntu8)`. +func systemdVersion(firstLine string) (int, bool) { + fields := strings.Fields(firstLine) + if len(fields) < 2 || fields[0] != "systemd" { + return 0, false + } + n, err := strconv.Atoi(fields[1]) + return n, err == nil +} +``` + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine` +Expected: PASS. Existing runner tests that assert `run --rm --no-deps --name 'sample-nightly-1'` must be updated to `run --rm --no-deps "$@" --name 'sample-nightly-1'` (TestScheduledJobUnitContract and TestPinnedScheduledJobLockProtocol). + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine +git commit -m "feat(schedule): manual activations consume declared inputs; hosts need systemd 252 for them + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 10: `Engine.ScheduleRun` and the `schedule_run` operation + +**Files:** +- Create: `internal/engine/schedule_run.go` +- Modify: `internal/onebox/operation_types.go:43,480` (`KindScheduleRun`), `internal/onebox/execution_types.go:288-310,357-373` (`ExecuteRequest.Job/Inputs/Wait`, `OperationResult.ScheduleRun`), `internal/onebox/execute.go:208` (dispatch), `internal/onebox/binding.go:34`, `internal/engine/audit.go:33-57` (`schedule-run` action and outcome) +- Test: `internal/engine/schedule_run_test.go` + +**Interfaces:** +- Produces: + - `type ScheduleRunResult struct { Job, Unit, Operation string; Inputs map[string]string; Started bool; Record *ScheduleRunRecord }` JSON `job, unit, operation, inputs, started, record`. + - `func (e *Engine) ScheduleRun(ctx, operationID, job string, inputs map[string]string, wait bool) (ScheduleRunResult, error)`. + - `KindScheduleRun OperationKind = "schedule_run"`; `ExecuteRequest.Job string`, `ExecuteRequest.Inputs map[string]string`, `ExecuteRequest.Wait bool`; `OperationResult.ScheduleRun *engine.ScheduleRunResult json:"schedule_run,omitempty"`. + - Journal: phase `schedule-run`, `Target: job`, `Detail: "inputs: K=V,..."` (or `inputs: defaults`), events `start` then `finish` (`Status: ok`, `Detail: "unit started; outcome in ob schedule history"`), both before the lock is released; `auditAction("schedule-run") == "schedule run"`, `auditOutcome("schedule run") == "started"`. + +- [ ] **Step 1: Write the failing test** + +```go +package engine + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/transport" +) + +func TestScheduleRunWritesInputsJournalsThenStartsAfterReleasingTheLock(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog", "prices"}, Default: "catalog"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{ExitCode: 3}, true + case strings.Contains(cmd, "systemctl start"): + return transport.Result{}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + result, err := e.ScheduleRun(context.Background(), "20260905-151200-schedule_run-7c1e", "sync", map[string]string{"SOURCE": "prices"}, false) + if err != nil { + t.Fatal(err) + } + if !result.Started || result.Unit != "ob-sample-sync" || result.Inputs["SOURCE"] != "prices" { + t.Fatalf("result = %#v", result) + } + seq := strings.Join(f.Commands, "\n") + inputs := strings.Index(seq, "sync.inputs") + start := strings.Index(seq, "systemctl start --no-block 'ob-sample-sync.service'") + release := strings.LastIndex(seq, "rm -f '/var/lib/ob/sample/lock'") + if inputs < 0 || start < 0 || release < 0 || !(inputs < release && release < start) { + t.Fatalf("expected inputs write, lock release, then start:\n%s", seq) + } + if !strings.Contains(seq, "set -C") { + t.Fatalf("inputs file was not created with noclobber:\n%s", seq) + } + written := strings.Join(f.Inputs, "\n") + for _, want := range []string{"ONEBOX_OPERATION=20260905-151200-schedule_run-7c1e", "SOURCE=prices"} { + if !strings.Contains(written, want) { + t.Fatalf("inputs file is missing %q:\n%s", want, written) + } + } + if !strings.Contains(written, `"phase":"schedule-run"`) || !strings.Contains(written, `"target":"sync"`) { + t.Fatalf("schedule run was not journaled:\n%s", written) + } +} + +func TestScheduleRunRefusals(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog"}, Default: "catalog"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + cfg.Workloads["prune"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "destructive", + Schedule: &app.JobSchedule{Cron: "0 3 * * *", Timezone: "UTC", Timeout: "1h"}, + } + active := false + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl is-active") { + if active { + return transport.Result{Stdout: "activating\n"}, true + } + return transport.Result{ExitCode: 3}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + ctx := context.Background() + if _, err := e.ScheduleRun(ctx, "op", "prune", nil, false); err == nil || !strings.Contains(err.Error(), "ob job run") { + t.Fatalf("destructive job accepted: %v", err) + } + if _, err := e.ScheduleRun(ctx, "op", "sync", map[string]string{"SOURCE": "reviews"}, false); err == nil { + t.Fatal("undeclared value accepted") + } + if _, err := e.ScheduleRun(ctx, "op", "web", nil, false); err == nil { + t.Fatal("non-scheduled workload accepted") + } + active = true + if _, err := e.ScheduleRun(ctx, "op", "sync", nil, false); err == nil || !strings.Contains(err.Error(), "running") { + t.Fatalf("active unit not refused: %v", err) + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/engine -run 'TestScheduleRun'` +Expected: FAIL to compile. + +- [ ] **Step 3: Implement `internal/engine/schedule_run.go`** + +```go +package engine + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/journal" +) + +// ScheduleRunResult is what an operator-initiated run leaves on the +// workstation side. The outcome lives on the host: it is the run record. +type ScheduleRunResult struct { + Job string `json:"job"` + Unit string `json:"unit"` + Operation string `json:"operation"` + Inputs map[string]string `json:"inputs,omitempty"` + Started bool `json:"started"` + Record *ScheduleRunRecord `json:"record,omitempty"` +} + +// ScheduleRun starts a scheduled job's unit now with declared, validated +// inputs. It is journaled like every operation, but the application lock is +// released before the unit starts: the runner exits 75 when it sees that lock. +func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inputs map[string]string, wait bool) (ScheduleRunResult, error) { + result := ScheduleRunResult{Job: name, Operation: operationID, Inputs: inputs} + if strings.TrimSpace(operationID) == "" { + return result, errors.New("schedule run requires an operation id") + } + if err := e.RequireHostOwner(ctx); err != nil { + return result, err + } + job, err := e.scheduledJob(name) + if err != nil { + return result, err + } + workload := e.Spec.Workloads[name] + if workload.DataEffect != app.DataEffectNone { + return result, fmt.Errorf("job %s declares data_effect %q; operator-initiated runs of it go through the sealed plan: ob job plan %s, then ob job run", name, workload.DataEffect, name) + } + if err := app.ValidateJobInputValues(workload, inputs); err != nil { + return result, err + } + unit := e.names().ScheduledJobUnit(name) + result.Unit = unit + + active, err := e.T.Run(ctx, "systemctl is-active "+q(unit+".service")+" 2>/dev/null || true") + if err != nil { + return result, err + } + if state := strings.TrimSpace(active.Stdout); state == "active" || state == "activating" || state == "deactivating" { + return result, fmt.Errorf("job %s is running (%s); wait for it or read ob schedule history %s", name, state, name) + } + + epoch, err := e.AcquireLock(ctx, operationID, e.Opts.ForceLock) + if err != nil { + return result, err + } + locked := true + defer func() { + if locked { + e.ReleaseLock(ctx) + } + }() + if err := e.WriteFence(ctx, operationID, epoch); err != nil { + return result, err + } + + body := scheduleInputsFile(operationID, inputs) + path := e.names().ScheduledJobRunInputs(name) + create := "umask 077 && install -d -m 700 " + q(e.names().AppDir()+"/schedule") + " && set -C && cat > " + q(path) + res, err := e.T.RunInput(ctx, create, body) + if err != nil { + return result, err + } + if res.ExitCode != 0 { + return result, fmt.Errorf("a manual run of %s is already pending (%s exists); wait for it or remove the file on the host", name, path) + } + + writer := &journal.Writer{ + T: e.T, Names: e.names(), DeployID: operationID, Epoch: epoch, Operator: journal.DefaultOperator(), + GitSHA: e.Opts.GitSHA, ConfigHash: e.Opts.ConfigHash, Runner: &e.Opts.Runner, + } + detail := "inputs: defaults" + if len(inputs) > 0 { + detail = "inputs: " + scheduleInputsDetail(inputs) + } + record := journal.Record{Phase: "schedule-run", Event: "start", Status: "ok", Target: name, TargetKind: "job", Detail: detail} + if err := writer.Append(ctx, record); err != nil { + return result, fmt.Errorf("journal schedule run start: %w", err) + } + record.Event, record.Detail = "finish", "unit started; outcome in ob schedule history "+name + if err := writer.Append(ctx, record); err != nil { + return result, fmt.Errorf("journal schedule run finish: %w", err) + } + e.ReleaseLock(ctx) + locked = false + + start := "systemctl start --no-block " + q(unit+".service") + if wait { + start = "systemctl start " + q(unit+".service") + } + res, err = e.mutate(ctx, start) + if err != nil { + return result, err + } + result.Started = true + if res.ExitCode != 0 && !wait { + return result, fmt.Errorf("systemctl start %s: %s", unit, strings.TrimSpace(res.Stderr)) + } + if wait { + records, err := e.ScheduleHistory(ctx, name, 1) + if err != nil { + return result, err + } + if len(records) > 0 { + result.Record = &records[0] + } + if res.ExitCode != 0 && (result.Record == nil || result.Record.Outcome != "skipped") { + return result, fmt.Errorf("job %s did not succeed; see ob schedule logs %s", name, name) + } + } + _ = job + return result, nil +} + +func scheduleInputsFile(operationID string, inputs map[string]string) string { + lines := []string{app.ReservedInputPrefix + "OPERATION=" + operationID} + for _, name := range sortedInputNames(inputs) { + lines = append(lines, name+"="+inputs[name]) + } + return strings.Join(lines, "\n") + "\n" +} + +func scheduleInputsDetail(inputs map[string]string) string { + parts := make([]string, 0, len(inputs)) + for _, name := range sortedInputNames(inputs) { + parts = append(parts, name+"="+inputs[name]) + } + return strings.Join(parts, ",") +} + +func sortedInputNames(inputs map[string]string) []string { + names := make([]string, 0, len(inputs)) + for name := range inputs { + names = append(names, name) + } + sort.Strings(names) + return names +} +``` + +`e.mutate` is fence-checked; if it refuses commands that are not file writes, use `e.T.Run` for the start. Check `internal/engine/lock.go:312` (`mutate`) first and use whichever the existing `systemctl enable --now` path uses (it uses `e.mutate`, so `mutate` is right; but after `ReleaseLock` the fence value is still ours, so it passes). + +Wiring in `internal/onebox`: + +```go +// operation_types.go + KindScheduleRun OperationKind = "schedule_run" +``` + +add `KindScheduleRun` to `validOperationKind` and `operationUsesInspectionRuntime`; add to `ExecuteRequest`: + +```go + // Job, Inputs and Wait are the schedule_run arguments: a declared + // scheduled job, validated input overrides, and whether to block until + // the unit exits. + Job string + Inputs map[string]string + Wait bool +``` + +`OperationResult`: `ScheduleRun *engine.ScheduleRunResult json:"schedule_run,omitempty"`. `execute.go` dispatch: + +```go + case KindScheduleRun: + result.EvidenceID = operationID + var run engine.ScheduleRunResult + run, err = e.ScheduleRun(ctx, operationID, request.Job, request.Inputs, request.Wait) + result.ScheduleRun = &run +``` + +`audit.go`: `case "schedule-run": return "schedule run"` and `case "schedule run": return "started"`. + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine ./internal/onebox` +Expected: PASS. If an `internal/onebox` test enumerates every kind (search `validOperationKind` in tests), add `KindScheduleRun` there. + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine/schedule_run.go internal/engine/schedule_run_test.go internal/engine/audit.go internal/onebox +git commit -m "feat(schedule): operator-initiated runs with validated inputs, journaled as schedule_run + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 11: CLI `ob schedule run` + +**Files:** +- Modify: `cmd/ob/schedule.go`, `cmd/ob/output.go` (matrix), `cmd/ob/output_test.go` (wantMatrix), `site/src/content/docs/reference/policies.mdx` +- Test: `cmd/ob/output_test.go`, `cmd/ob/schedule_test.go` + +**Interfaces:** +- Consumes: `runMutation(cmd, g, onebox.ExecuteRequest{Kind: onebox.KindScheduleRun, Job, Inputs, Wait, BreakLock}, "schedule run")`. +- Produces matrix row `"ob schedule run": {finite_stream, JSON, NDJSON}`; flag parsing `--input NAME=VALUE` (repeatable) into `map[string]string`, refusing a duplicate name or a missing `=` before any connection. + +- [ ] **Step 1: Write the failing tests** + +`cmd/ob/schedule_test.go`: + +```go +package main + +import "testing" + +func TestParseScheduleInputsFlags(t *testing.T) { + got, err := parseScheduleInputs([]string{"SOURCE=prices", "SINCE=2026-09-01"}) + if err != nil || got["SOURCE"] != "prices" || got["SINCE"] != "2026-09-01" { + t.Fatalf("got %#v, %v", got, err) + } + for _, bad := range [][]string{{"SOURCE"}, {"=x"}, {"SOURCE=a", "SOURCE=b"}} { + if _, err := parseScheduleInputs(bad); err == nil { + t.Errorf("%v was accepted", bad) + } + } +} +``` + +Add `"ob schedule run": {Class: "finite_stream", JSON: true, NDJSON: true}` to `wantMatrix`. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./cmd/ob -run 'TestParseScheduleInputsFlags|TestLeafOutputMatrix'` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +```go +func parseScheduleInputs(raw []string) (map[string]string, error) { + out := map[string]string{} + for _, item := range raw { + name, value, ok := strings.Cut(item, "=") + if !ok || name == "" { + return nil, fmt.Errorf("--input %q must be NAME=VALUE", item) + } + if _, dup := out[name]; dup { + return nil, fmt.Errorf("--input %s given twice", name) + } + out[name] = value + } + return out, nil +} +``` + +Command: + +```go + var runInputs []string + var runWait, runBreakLock bool + runCmd := &cobra.Command{ + Use: "run ", + Short: "start a scheduled job now with declared inputs", + Long: "Start one scheduled job's unit now, with values for its declared inputs. Values are validated on the workstation against the declaration; an undeclared name or a value outside its enum or pattern is refused before anything reaches the host.\n\nOnly a job with data_effect none may run this way; a migration or destructive job keeps the sealed plan of ob job run. The request is journaled as schedule_run with the operator and inputs. The outcome is the run record: ob schedule history , or --wait to block until the unit exits and print it.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + inputs, err := parseScheduleInputs(runInputs) + if err != nil { + return writeEarlyOperationFailure(cmd, g, codedError("invalid_argument", "%v", err)) + } + return runMutation(cmd, g, onebox.ExecuteRequest{ + Kind: onebox.KindScheduleRun, Job: args[0], Inputs: inputs, Wait: runWait, BreakLock: runBreakLock, + }, "schedule run") + }, + } + runCmd.Flags().StringArrayVar(&runInputs, "input", nil, "input override as NAME=VALUE; repeatable") + runCmd.Flags().BoolVar(&runWait, "wait", false, "block until the unit exits and print the run record") + runCmd.Flags().BoolVar(&runBreakLock, "break-lock", false, "break a stale operation lock after inspecting its holder") + scheduleCmd.AddCommand(runCmd) +``` + +Check `codedError`'s signature at `cmd/ob/job.go:182` usage and match it. After a successful human-mode run, `runMutation` prints the operation outcome; the record is in `result.ScheduleRun` for structured output. For human output with `--wait`, print the record after `runMutation` returns nil: `result` is not returned by `runMutation`, so add an optional `onResult func(onebox.OperationResult)` hook only if `runMutation` already exposes one; otherwise leave the human hint `run: ob schedule history ` in the command's `Long` and rely on structured output for the record. + +Add `` `ob schedule run` `` to the Finite operation stream row in `policies.mdx`. + +- [ ] **Step 4: Regenerate docs and run tests** + +Run: `just build && just docs-generate && go test ./... && just docs-generate-check && just env-namespace` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add cmd/ob site/src/content/docs/reference +git commit -m "feat(cli): ob schedule run starts a scheduled job now with validated inputs + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +## Docs and end-to-end + +### Task 12: Guide, capabilities page, issue follow-up + +**Files:** +- Modify: `site/src/content/docs/guides/schedule-a-job.mdx`, `site/src/content/docs/status/capabilities.mdx` +- Regenerate: `site/src/content/docs/reference/fields/workloads.mdx`, `site/src/content/docs/reference/cli.mdx`, `site/public/onebox.run-v1.schema.json`, `docs/onebox.run-v1.schema.json` + +- [ ] **Step 1: Guide sections** + +In `schedule-a-job.mdx`: +- Replace the sentence "A failed run is not retried implicitly; the next attempt is the next cron elapse." with a `## Retry inside one firing` section: the `retry` block, attempts counting, doubling capped by `max_backoff`, the timeout bound, what exclusive and pinned hold during backoff, and that the next cron elapse is still the retry for short cadences. +- Rewrite `## Failures remain visible` as `## Every run leaves a record`: the record fields, `ob schedule history `, `ob schedule logs [--run ]`, `ob schedule list`, the new `ob status` line, `skipped` runs, journal retention and the volatile-journal note, and `notify`. +- Add `## Run one now with inputs` before `## Running one by hand`: the `inputs` block, naming and value rules, `ob schedule run --input NAME=VALUE --wait`, the `data_effect: none` rule with the pointer to `ob job run`, the `TRIGGER_UNIT` requirement (systemd 252, Ubuntu 24.04 and Debian 12), and that the request appears in `ob audit` joined to the record by operation id. +- Update the frontmatter `description`/`summary` to mention retry, run history and manual runs. + +- [ ] **Step 2: Capabilities page** + +Under `## Shipped`, add one bullet: + +```markdown +- Scheduled jobs with bounded retry inside one firing, one run record per + activation in the host journal (`ob schedule history`, `ob schedule logs`, + `ob status`), per-outcome notifications, and declared inputs for + operator-initiated runs (`ob schedule run`). +``` + +- [ ] **Step 3: Regenerate and verify** + +Run: `just build && just docs-generate && go run ./cmd/ob schema --out docs/onebox.run-v1.schema.json && just check` +Expected: PASS (`just check` runs mod-tidy, fmt-check, vet, test, docs-generate-check, site-build; `site-build` needs `just site-install` once). + +- [ ] **Step 4: Commit** + +```bash +git add site docs +git commit -m "docs(site): retry, run history, notify and manual runs for scheduled jobs + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +- [ ] **Step 5: Issue follow-up** + +Edit #155: `OB_` becomes `ONEBOX_` (reserved prefix and metadata line), the systemd floor check lives in `SyncSchedules`, the volatile journal is a status note rather than an issue, and `ob schedule run` journals start and finish before releasing the lock. Post one comment summarising the deviations. + +### Task 13: End-to-end coverage on the Lima host + +**Files:** +- Modify: `e2e/testdata/postgres/ob.yml.tmpl:37-51`, `e2e/server_test.go:188-292` + +- [ ] **Step 1: Fixture jobs** + +Add beside `timeout-chore`: + +```yaml + # Fails once, then succeeds: proves the in-firing retry and that the run + # record counts attempts. The marker lives on the host so the second + # attempt, a fresh container, can see the first one ran. + retry-chore: + role: job + image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 + command: ["sh", "-c", "if [ -f /marker/ran ]; then echo second; else touch /marker/ran; exit 1; fi"] + data_effect: none + volumes: [{source: /tmp/onebox-e2e-retry, path: /marker}] + schedule: { cron: "0 0 1 1 *", timeout: 60s, catch_up: false, retry: {attempts: 2, backoff: 1s} } + # A declared input reaches the container as an environment variable, both + # as its default on a timer-shaped start and as an override on a manual run. + input-chore: + role: job + image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 + command: ["sh", "-c", "echo greeting=$GREETING"] + data_effect: none + inputs: + GREETING: {enum: [hi, hello], default: hi} + schedule: { cron: "0 0 1 1 *", timeout: 20s, catch_up: false } +``` + +- [ ] **Step 2: Assertions** + +In the "scheduled jobs" subtest after the normal `chore` run: + +```go + history := s.mustOb(t, dir, "schedule", "history", "chore", "--output", "json") + for _, want := range []string{`"outcome":"success"`, `"trigger":"manual"`, `"attempts":1`} { + if !strings.Contains(history, want) { + t.Fatalf("history is missing %q:\n%s", want, history) + } + } + s.run(t, "rm -rf /tmp/onebox-e2e-retry && mkdir -p /tmp/onebox-e2e-retry") + s.run(t, "systemctl start ob-observer-retry--chore.service") + retry := s.mustOb(t, dir, "schedule", "history", "retry-chore", "--output", "json") + for _, want := range []string{`"outcome":"success"`, `"attempts":2`} { + if !strings.Contains(retry, want) { + t.Fatalf("retry history is missing %q:\n%s", want, retry) + } + } + manual := s.mustOb(t, dir, "schedule", "run", "input-chore", "--input", "GREETING=hello", "--wait", "--output", "json") + if !strings.Contains(manual, `"GREETING":"hello"`) || !strings.Contains(manual, `"outcome":"success"`) { + t.Fatalf("manual run record missing inputs or outcome:\n%s", manual) + } + logs := s.mustOb(t, dir, "schedule", "logs", "input-chore") + if !strings.Contains(logs, "greeting=hello") { + t.Fatalf("run logs do not show the override:\n%s", logs) + } + list := s.mustOb(t, dir, "schedule", "list") + if !strings.Contains(list, "input-chore") || !strings.Contains(list, "active") { + t.Fatalf("schedule list did not show the timer:\n%s", list) + } +``` + +Add to the timeout section: `"last run failed: timeout"` stays; the record for `timeout-chore` shows `"outcome":"timeout"` via `ob schedule history timeout-chore --output json`. + +Check the unit name escaping for `retry-chore` (`ob-observer-retry--chore`, the double hyphen is how `timeout-chore` is spelled at line 192). + +- [ ] **Step 3: Run** + +Run: `command -v limactl && just server-e2e` (boots Ubuntu 24.04, about a minute, then the suite). If Lima is not installed, record that the e2e suite was not run locally and rely on CI. + +- [ ] **Step 4: Commit** + +```bash +git add e2e +git commit -m "test(e2e): scheduled-job retry, run records, manual runs with inputs on a real host + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +## Self-review notes + +- Spec coverage: history (Tasks 1-5), retry and notify (6-7), inputs and manual runs (8-11), docs (12), host validation (9, 13). `ob doctor` is not touched: the journal note lives in `ob status`, and the systemd floor in `SyncSchedules`, both recorded as deviations in Task 12 Step 5. +- Type consistency: `ScheduleRunRecord`, `ScheduleListing`, `ScheduleRunResult`, `JobRetry`, `JobInput`, `KindScheduleRun`, `scheduleRunIdentifier`, `scheduleHistoryCommand`, `scheduleInputsLines`, `scheduleAttemptLoop`, `scheduleNotifier` are used with the same names throughout. From 92f0a4bd6294c0bde8291f7a180fda39280d89f2 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:16:27 -0700 Subject: [PATCH 06/25] feat(cli): ob schedule list, history and logs read the host's run records Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- cmd/ob/ops.go | 24 +-- cmd/ob/output.go | 77 ++++---- cmd/ob/output_test.go | 77 ++++---- cmd/ob/schedule.go | 181 +++++++++++++++++++ internal/onebox/operation_errors.go | 12 ++ site/src/content/docs/reference/cli.mdx | 68 ++++++- site/src/content/docs/reference/errors.mdx | 3 + site/src/content/docs/reference/policies.mdx | 4 +- 8 files changed, 347 insertions(+), 99 deletions(-) create mode 100644 cmd/ob/schedule.go diff --git a/cmd/ob/ops.go b/cmd/ob/ops.go index bc68060..1a2b28a 100644 --- a/cmd/ob/ops.go +++ b/cmd/ob/ops.go @@ -64,28 +64,8 @@ func addOpsCommands(root *cobra.Command, g *globalFlags) { proxyCmd.AddCommand(proxyApplyCmd) root.AddCommand(proxyCmd) - // schedule apply — explicitly reconcile runner-owned host timers after an - // Onebox upgrade, without coupling package installation to remote mutation. - scheduleCmd := &cobra.Command{Use: "schedule", Short: "manage host timers for scheduled jobs", - Long: "Manage the systemd timers generated for scheduled jobs.\n\n" + - "Timers outlive the Onebox process and the package installed on the operator\n" + - "workstation. `apply` explicitly reconciles their units after a runner or\n" + - "configuration change without deploying a release.", - Args: cobra.NoArgs, RunE: showCommandHelp} - var scheduleBreakLock bool - scheduleApplyCmd := &cobra.Command{ - Use: "apply", - Short: "reconcile scheduled-job units without deploying a release", - Long: "Converge every declared scheduled-job timer, service, runner, and failure notifier to what the current Onebox runner generates.\n\nTaken under the application lock and fence so a deploy or host-fired job cannot modify the same runtime concurrently. This is the explicit post-upgrade path; upgrading the local package never mutates a remote host by itself.", - RunE: func(cmd *cobra.Command, _ []string) error { - return runMutation(cmd, g, onebox.ExecuteRequest{ - Kind: onebox.KindScheduleApply, BreakLock: scheduleBreakLock, - }, "schedule apply") - }, - } - scheduleApplyCmd.Flags().BoolVar(&scheduleBreakLock, "break-lock", false, "break a stale operation lock after inspecting its holder") - scheduleCmd.AddCommand(scheduleApplyCmd) - root.AddCommand(scheduleCmd) + // schedule apply | list | history | logs — see schedule.go. + addScheduleCommands(root, g) // secrets list | edit | push secretsCmd := &cobra.Command{Use: "secrets", Short: "SOPS-encrypted secrets", diff --git a/cmd/ob/output.go b/cmd/ob/output.go index e6f4cd3..e1b9c4f 100644 --- a/cmd/ob/output.go +++ b/cmd/ob/output.go @@ -76,43 +76,46 @@ const ( // Cobra's native human output. Keeping this list closed makes adding a command // an explicit CLI-contract decision instead of silently inheriting behavior. var cliOutputMatrix = map[string]cliOutputClass{ - "ob abort": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob approve": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob audit": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob bootstrap": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob canonical": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob deploy": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob destroy": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob doctor": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob eject": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob exec": {Class: cliClassOperatorPassthrough, NDJSON: true}, - "ob init": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob backup create": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup enable": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup disable": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup drill": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup restore": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup prune": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup verify": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup status": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob job plan": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob job run": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob logs": {Class: cliClassOperatorPassthrough, JSON: true, NDJSON: true}, - "ob plan": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob preflight": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob preview": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob proxy apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob resume": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob rollback": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob schedule apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob schema": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob secrets edit": {Class: cliClassTrustedEditor, JSON: true}, - "ob secrets list": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob secrets push": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob service apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob status": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob validate": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob version": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob abort": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob approve": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob audit": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob bootstrap": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob canonical": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob deploy": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob destroy": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob doctor": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob eject": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob exec": {Class: cliClassOperatorPassthrough, NDJSON: true}, + "ob init": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob backup create": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup enable": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup disable": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup drill": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup restore": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup prune": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup verify": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup status": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob job plan": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob job run": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob logs": {Class: cliClassOperatorPassthrough, JSON: true, NDJSON: true}, + "ob plan": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob preflight": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob preview": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob proxy apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob resume": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob rollback": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob schedule apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob schedule history": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob schedule list": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob schedule logs": {Class: cliClassOperatorPassthrough, JSON: true, NDJSON: true}, + "ob schema": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob secrets edit": {Class: cliClassTrustedEditor, JSON: true}, + "ob secrets list": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob secrets push": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob service apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob status": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob validate": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob version": {Class: cliClassFiniteEnvelope, JSON: true}, } type cliExitError struct { diff --git a/cmd/ob/output_test.go b/cmd/ob/output_test.go index 3ded28d..88b2b23 100644 --- a/cmd/ob/output_test.go +++ b/cmd/ob/output_test.go @@ -490,43 +490,46 @@ workloads: func TestLeafOutputMatrixIsClosedAndHasNoAliases(t *testing.T) { wantMatrix := map[string]cliOutputClass{ - "ob abort": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob approve": {Class: "finite_envelope", JSON: true}, - "ob audit": {Class: "finite_envelope", JSON: true}, - "ob bootstrap": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob canonical": {Class: "finite_envelope", JSON: true}, - "ob deploy": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob destroy": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob doctor": {Class: "finite_envelope", JSON: true}, - "ob eject": {Class: "finite_envelope", JSON: true}, - "ob exec": {Class: "operator_passthrough", NDJSON: true}, - "ob init": {Class: "finite_envelope", JSON: true}, - "ob job plan": {Class: "finite_envelope", JSON: true}, - "ob backup create": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup enable": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup disable": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup drill": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup restore": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup prune": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup verify": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup status": {Class: "finite_envelope", JSON: true}, - "ob job run": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob logs": {Class: "operator_passthrough", JSON: true, NDJSON: true}, - "ob plan": {Class: "finite_envelope", JSON: true}, - "ob preflight": {Class: "finite_envelope", JSON: true}, - "ob preview": {Class: "finite_envelope", JSON: true}, - "ob proxy apply": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob resume": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob rollback": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob schedule apply": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob schema": {Class: "finite_envelope", JSON: true}, - "ob secrets edit": {Class: "trusted_editor", JSON: true}, - "ob secrets list": {Class: "finite_envelope", JSON: true}, - "ob secrets push": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob service apply": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob status": {Class: "finite_envelope", JSON: true}, - "ob validate": {Class: "finite_envelope", JSON: true}, - "ob version": {Class: "finite_envelope", JSON: true}, + "ob abort": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob approve": {Class: "finite_envelope", JSON: true}, + "ob audit": {Class: "finite_envelope", JSON: true}, + "ob bootstrap": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob canonical": {Class: "finite_envelope", JSON: true}, + "ob deploy": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob destroy": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob doctor": {Class: "finite_envelope", JSON: true}, + "ob eject": {Class: "finite_envelope", JSON: true}, + "ob exec": {Class: "operator_passthrough", NDJSON: true}, + "ob init": {Class: "finite_envelope", JSON: true}, + "ob job plan": {Class: "finite_envelope", JSON: true}, + "ob backup create": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup enable": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup disable": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup drill": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup restore": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup prune": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup verify": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup status": {Class: "finite_envelope", JSON: true}, + "ob job run": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob logs": {Class: "operator_passthrough", JSON: true, NDJSON: true}, + "ob plan": {Class: "finite_envelope", JSON: true}, + "ob preflight": {Class: "finite_envelope", JSON: true}, + "ob preview": {Class: "finite_envelope", JSON: true}, + "ob proxy apply": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob resume": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob rollback": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob schedule apply": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob schedule history": {Class: "finite_envelope", JSON: true}, + "ob schedule list": {Class: "finite_envelope", JSON: true}, + "ob schedule logs": {Class: "operator_passthrough", JSON: true, NDJSON: true}, + "ob schema": {Class: "finite_envelope", JSON: true}, + "ob secrets edit": {Class: "trusted_editor", JSON: true}, + "ob secrets list": {Class: "finite_envelope", JSON: true}, + "ob secrets push": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob service apply": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob status": {Class: "finite_envelope", JSON: true}, + "ob validate": {Class: "finite_envelope", JSON: true}, + "ob version": {Class: "finite_envelope", JSON: true}, } if !reflect.DeepEqual(cliOutputMatrix, wantMatrix) { t.Fatalf("CLI output matrix changed:\ngot %#v\nwant %#v", cliOutputMatrix, wantMatrix) diff --git a/cmd/ob/schedule.go b/cmd/ob/schedule.go new file mode 100644 index 0000000..ef7005f --- /dev/null +++ b/cmd/ob/schedule.go @@ -0,0 +1,181 @@ +package main + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "text/tabwriter" + + "github.com/labstack/onebox/internal/onebox" + "github.com/spf13/cobra" +) + +// addScheduleCommands wires `ob schedule`: apply reconciles the host units, +// list, history and logs read what the host recorded. The read commands +// connect directly, like `ob status`; they hold no lock and write nothing. +func addScheduleCommands(root *cobra.Command, g *globalFlags) { + scheduleCmd := &cobra.Command{Use: "schedule", Short: "manage host timers for scheduled jobs", + Long: "Manage the systemd timers generated for scheduled jobs.\n\n" + + "Timers outlive the Onebox process and the package installed on the operator\n" + + "workstation. `apply` explicitly reconciles their units after a runner or\n" + + "configuration change without deploying a release. `list`, `history` and `logs`\n" + + "read the timer state and the run records the host keeps in its journal.", + Args: cobra.NoArgs, RunE: showCommandHelp} + + var scheduleBreakLock bool + scheduleApplyCmd := &cobra.Command{ + Use: "apply", + Short: "reconcile scheduled-job units without deploying a release", + Long: "Converge every declared scheduled-job timer, service, runner, and failure notifier to what the current Onebox runner generates.\n\nTaken under the application lock and fence so a deploy or host-fired job cannot modify the same runtime concurrently. This is the explicit post-upgrade path; upgrading the local package never mutates a remote host by itself.", + RunE: func(cmd *cobra.Command, _ []string) error { + return runMutation(cmd, g, onebox.ExecuteRequest{ + Kind: onebox.KindScheduleApply, BreakLock: scheduleBreakLock, + }, "schedule apply") + }, + } + scheduleApplyCmd.Flags().BoolVar(&scheduleBreakLock, "break-lock", false, "break a stale operation lock after inspecting its holder") + scheduleCmd.AddCommand(scheduleApplyCmd) + + listCmd := &cobra.Command{ + Use: "list", + Short: "declared scheduled jobs with timer state and next elapse", + Long: "List every job that declares a schedule beside what the host's timer says: whether it is active, when it fires next, and when it last fired. Reads only.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, p, err := loadAllLenient(cmd.Context(), g) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + defer cleanup() + jobs, err := e.ScheduleList(cmd.Context()) + if err != nil { + return writeStructuredCommandFailure(cmd, g, "schedule_list_failed", "scheduled jobs could not be listed", err) + } + if isStructuredOutput(g) { + return writeFiniteSuccess(cmd, g, map[string]any{"jobs": jobs}) + } + if len(jobs) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "no scheduled jobs declared") + return nil + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "JOB\tCRON\tTZ\tTIMER\tNEXT\tLAST TRIGGER\tPOLICY\tTIMEOUT") + for _, j := range jobs { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + j.Name, j.Cron, j.Timezone, orDash(j.TimerState), orDash(j.NextRun), orDash(j.LastTrigger), j.DeployLock, j.Timeout) + } + return w.Flush() + }, + } + scheduleCmd.AddCommand(listCmd) + + var historyCount int + historyCmd := &cobra.Command{ + Use: "history ", + Short: "run records of one scheduled job, newest first", + Long: "Read the run records the host wrote for one scheduled job. Each record is one activation: run id, trigger, release, start and end, attempts, exit status, outcome and, for a manual run, its inputs.\n\nRecords live in the host journal under the job's unit with syslog identifier ob-run; retention is the journal's. Reads only.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, p, err := loadAllLenient(cmd.Context(), g) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + defer cleanup() + records, err := e.ScheduleHistory(cmd.Context(), args[0], historyCount) + if err != nil { + return writeStructuredCommandFailure(cmd, g, "schedule_history_failed", "run history could not be read", err) + } + if isStructuredOutput(g) { + return writeFiniteSuccess(cmd, g, map[string]any{"job": args[0], "runs": records}) + } + if len(records) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "no recorded runs for %s\n", args[0]) + return nil + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "STARTED\tOUTCOME\tDURATION\tATTEMPTS\tEXIT\tTRIGGER\tRELEASE\tRUN") + for _, r := range records { + exit := "-" + if r.ExitStatus != nil { + exit = strconv.Itoa(*r.ExitStatus) + } + fmt.Fprintf(w, "%s\t%s\t%ds\t%d\t%s\t%s\t%s\t%s\n", + r.StartedAt, r.Outcome, r.DurationSeconds, r.Attempts, exit, r.Trigger, orDash(r.Release), r.Run) + } + return w.Flush() + }, + } + historyCmd.Flags().IntVarP(&historyCount, "count", "n", 20, "number of newest runs to show") + scheduleCmd.AddCommand(historyCmd) + + var logsRun string + var logsTail int + logsCmd := &cobra.Command{ + Use: "logs ", + Short: "journal of one scheduled run", + Long: "Stream the host journal for one run of a scheduled job: by default the newest recorded run, or the run named with --run. The run id is systemd's invocation id, so the output is exactly that activation. Reads only.\n\nLog bytes are operator-controlled and may contain secrets; Onebox does not claim\nto redact passthrough output.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, p, err := loadAllLenient(cmd.Context(), g) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + defer cleanup() + if g.Output == "json" { + var stdout, stderr bytes.Buffer + err = e.ScheduleLogs(cmd.Context(), args[0], logsRun, logsTail, &stdout, &stderr) + data := map[string]any{ + "job": args[0], "run": logsRun, "stdout": stdout.String(), "stderr": stderr.String(), + "passthrough_unredacted": true, + } + if err != nil { + publicErr := publicError(err, "schedule_logs_failed", "run logs could not be read") + publicErr.Details = data + if writeErr := writeFiniteOutcome(cmd, g, cliOutcomeError, nil, publicErr); writeErr != nil { + return writeErr + } + return withExitCode(err, 1) + } + return writeFiniteSuccess(cmd, g, data) + } + if g.Output == "ndjson" { + stream := newCLIRecordStream(cmd.OutOrStdout(), commandName(cmd)) + err = e.ScheduleLogs(cmd.Context(), args[0], logsRun, logsTail, stream.channelWriter("stdout"), stream.channelWriter("stderr")) + data := map[string]any{"job": args[0], "run": logsRun, "passthrough_unredacted": true} + if err != nil { + if writeErr := stream.terminal(cliOutcomeError, nil, publicError(err, "schedule_logs_failed", "run logs could not be read")); writeErr != nil { + return writeErr + } + return withExitCode(err, 1) + } + return stream.terminal(cliOutcomeSuccess, data, nil) + } + return e.ScheduleLogs(cmd.Context(), args[0], logsRun, logsTail, cmd.OutOrStdout(), cmd.ErrOrStderr()) + }, + } + logsCmd.Flags().StringVar(&logsRun, "run", "", "run id from ob schedule history; default the newest run") + logsCmd.Flags().IntVarP(&logsTail, "tail", "n", 200, "lines to show when no run is recorded") + scheduleCmd.AddCommand(logsCmd) + + root.AddCommand(scheduleCmd) +} + +func orDash(s string) string { + if strings.TrimSpace(s) == "" { + return "-" + } + return s +} diff --git a/internal/onebox/operation_errors.go b/internal/onebox/operation_errors.go index b5f3c32..1548065 100644 --- a/internal/onebox/operation_errors.go +++ b/internal/onebox/operation_errors.go @@ -121,6 +121,18 @@ var operationFailureDefinitions = map[string]OperationFailure{ Message: "a release manifest is not valid closed JSON for its schema", Command: "ob status --output json", }, + "schedule_history_failed": { + Message: "the scheduled job's run records could not be read from the host journal", + Command: "ob status --output json", + }, + "schedule_list_failed": { + Message: "the scheduled jobs' timer state could not be read", + Command: "ob status --output json", + }, + "schedule_logs_failed": { + Message: "the scheduled run's journal could not be read", + Command: "ob schedule history --output json", + }, "manifest_missing": { Message: "a release directory carries no manifest, so its lifecycle state is unknown", Command: "ob status --output json", diff --git a/site/src/content/docs/reference/cli.mdx b/site/src/content/docs/reference/cli.mdx index 08c6935..c01cc07 100644 --- a/site/src/content/docs/reference/cli.mdx +++ b/site/src/content/docs/reference/cli.mdx @@ -929,7 +929,8 @@ Manage the systemd timers generated for scheduled jobs. Timers outlive the Onebox process and the package installed on the operator workstation. `apply` explicitly reconciles their units after a runner or -configuration change without deploying a release. +configuration change without deploying a release. `list`, `history` and `logs` +read the timer state and the run records the host keeps in its journal. Usage: ob schedule [flags] @@ -937,6 +938,9 @@ Usage: Available Commands: apply reconcile scheduled-job units without deploying a release + history run records of one scheduled job, newest first + list declared scheduled jobs with timer state and next elapse + logs journal of one scheduled run Flags: -h, --help help for schedule @@ -971,6 +975,68 @@ Global Flags: -v, --verbose print every remote command ``` +### ob schedule history + +``` +Read the run records the host wrote for one scheduled job. Each record is one activation: run id, trigger, release, start and end, attempts, exit status, outcome and, for a manual run, its inputs. + +Records live in the host journal under the job's unit with syslog identifier ob-run; retention is the journal's. Reads only. + +Usage: + ob schedule history [flags] + +Flags: + -n, --count int number of newest runs to show (default 20) + -h, --help help for history + +Global Flags: + -c, --config string path to the project YAML file (default "ob.yml") + -e, --env string environment name (default "production") + --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") + -v, --verbose print every remote command +``` + +### ob schedule list + +``` +List every job that declares a schedule beside what the host's timer says: whether it is active, when it fires next, and when it last fired. Reads only. + +Usage: + ob schedule list [flags] + +Flags: + -h, --help help for list + +Global Flags: + -c, --config string path to the project YAML file (default "ob.yml") + -e, --env string environment name (default "production") + --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") + -v, --verbose print every remote command +``` + +### ob schedule logs + +``` +Stream the host journal for one run of a scheduled job: by default the newest recorded run, or the run named with --run. The run id is systemd's invocation id, so the output is exactly that activation. Reads only. + +Log bytes are operator-controlled and may contain secrets; Onebox does not claim +to redact passthrough output. + +Usage: + ob schedule logs [flags] + +Flags: + -h, --help help for logs + --run string run id from ob schedule history; default the newest run + -n, --tail int lines to show when no run is recorded (default 200) + +Global Flags: + -c, --config string path to the project YAML file (default "ob.yml") + -e, --env string environment name (default "production") + --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") + -v, --verbose print every remote command +``` + ## ob schema ``` diff --git a/site/src/content/docs/reference/errors.mdx b/site/src/content/docs/reference/errors.mdx index 99fe77b..21b585c 100644 --- a/site/src/content/docs/reference/errors.mdx +++ b/site/src/content/docs/reference/errors.mdx @@ -152,6 +152,9 @@ step to complete rather than a line to run verbatim. | `preflight_failed` | a target readiness check failed before any mutation | — | — | | `recovery_incomplete` | recovery did not reach its verified terminal state | resolving | `ob resume --output ndjson` | | `rollback_target_missing` | no previously serving release is recorded as a rollback target | next | `ob plan --output json` | +| `schedule_history_failed` | the scheduled job's run records could not be read from the host journal | diagnostic | `ob status --output json` | +| `schedule_list_failed` | the scheduled jobs' timer state could not be read | diagnostic | `ob status --output json` | +| `schedule_logs_failed` | the scheduled run's journal could not be read | next | `ob schedule history --output json` | | `secret_cleanup_pending` | the rotation is applied and verified, but removing the retired generation did not finish | resolving | `ob secrets push --output ndjson` | | `secret_declaration_not_deployed` | the deployed release does not declare this secret graph | next | `ob plan --output json` | | `secret_entry_not_selected` | more than one editable secret source exists, so an entry identifier is required | diagnostic | `ob secrets list --output json` | diff --git a/site/src/content/docs/reference/policies.mdx b/site/src/content/docs/reference/policies.mdx index 16f0791..03ba005 100644 --- a/site/src/content/docs/reference/policies.mdx +++ b/site/src/content/docs/reference/policies.mdx @@ -96,9 +96,9 @@ redacted. | Class | JSON | NDJSON | Commands | | --- | --- | --- | --- | -| Finite envelope | yes | no | `ob approve` · `ob audit` · `ob backup status` · `ob canonical` · `ob doctor` · `ob eject` · `ob init` · `ob job plan` · `ob plan` · `ob preflight` · `ob preview` · `ob schema` · `ob secrets list` · `ob status` · `ob validate` · `ob version` | +| Finite envelope | yes | no | `ob approve` · `ob audit` · `ob backup status` · `ob canonical` · `ob doctor` · `ob eject` · `ob init` · `ob job plan` · `ob plan` · `ob preflight` · `ob preview` · `ob schedule history` · `ob schedule list` · `ob schema` · `ob secrets list` · `ob status` · `ob validate` · `ob version` | | Finite operation stream | yes | yes | `ob abort` · `ob backup create` · `ob backup enable` · `ob backup disable` · `ob backup drill` · `ob backup prune` · `ob backup restore` · `ob backup verify` · `ob bootstrap` · `ob deploy` · `ob destroy` · `ob job run` · `ob proxy apply` · `ob resume` · `ob rollback` · `ob schedule apply` · `ob secrets push` · `ob service apply` | -| Operator passthrough | finite only | yes | `ob logs` | +| Operator passthrough | finite only | yes | `ob logs` · `ob schedule logs` | | Operator passthrough | no | yes | `ob exec` | | Trusted editor | yes, after exit | no | `ob secrets edit` | From 26035dc059d24a6d0233d940bb87e429c58e2dc3 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:19:48 -0700 Subject: [PATCH 07/25] feat(schema): schedule.retry and schedule.notify with validation against the timeout Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- docs/onebox.run-v1.schema.json | 42 +++++++ internal/app/constraints.go | 1 + internal/app/schedule.go | 10 ++ internal/app/schedule_retry.go | 105 ++++++++++++++++++ internal/app/schedule_test.go | 76 +++++++++++++ internal/app/types.go | 21 +++- internal/app/validate.go | 3 + site/public/onebox.run-v1.schema.json | 42 +++++++ .../docs/reference/fields/workloads.mdx | 7 +- 9 files changed, 301 insertions(+), 6 deletions(-) create mode 100644 internal/app/schedule_retry.go diff --git a/docs/onebox.run-v1.schema.json b/docs/onebox.run-v1.schema.json index ef54090..9981c67 100644 --- a/docs/onebox.run-v1.schema.json +++ b/docs/onebox.run-v1.schema.json @@ -2536,6 +2536,48 @@ ], "type": "string" }, + "notify": { + "default": "failure, timeout", + "description": "Run outcomes that send the configured notifications: success, failure, timeout, skipped.", + "items": { + "type": "string" + }, + "type": "array" + }, + "retry": { + "additionalProperties": false, + "description": "Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run.", + "patternProperties": { + "^x-": {} + }, + "properties": { + "attempts": { + "default": 1, + "description": "Total attempts including the first, 1 to 10.", + "examples": [ + 3 + ], + "type": "integer" + }, + "backoff": { + "default": "30s", + "description": "Sleep before the second attempt; it doubles after each failure.", + "examples": [ + "1m" + ], + "type": "string" + }, + "max_backoff": { + "default": "10m", + "description": "Upper bound for the doubling sleep.", + "examples": [ + "30m" + ], + "type": "string" + } + }, + "type": "object" + }, "timeout": { "default": "1h", "description": "Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d.", diff --git a/internal/app/constraints.go b/internal/app/constraints.go index 8742996..bd75732 100644 --- a/internal/app/constraints.go +++ b/internal/app/constraints.go @@ -171,6 +171,7 @@ var ( eMigrationPolicy = []string{"manual", "auto", "expand-only"} eNotifyFormat = []string{"text", "json"} eNotifyEvent = []string{"success", "failure"} + eScheduleNotify = []string{"success", "failure", "timeout", "skipped"} eProxyKind = []string{"traefik-docker", "none"} // One provider, because one is implemented. The withdrawn `secrets` block // accepted `age` and nothing ever decrypted it — every path shells out to diff --git a/internal/app/schedule.go b/internal/app/schedule.go index 531f1bd..fddeedb 100644 --- a/internal/app/schedule.go +++ b/internal/app/schedule.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" "strings" + "time" ) // A scheduled job is a promise that something runs at a time nobody will be @@ -32,6 +33,12 @@ type ScheduledJob struct { DeployLock string // Calendar is the host-side expression the cron translates to. Calendar string + // Retry and notify policy, resolved over the defaults so the runner and + // notifier renderers never see an absent block. + RetryAttempts int + RetryBackoff time.Duration + RetryMaxBackoff time.Duration + Notify []string } // ScheduledJobs lists every job with a schedule, in a stable order. @@ -55,9 +62,12 @@ func (p *Spec) ScheduledJobs() ([]ScheduledJob, error) { if deployLock == "" { deployLock = "exclusive" } + attempts, backoff, maxBackoff := w.Schedule.retryPolicy() out = append(out, ScheduledJob{ Name: name, Cron: w.Schedule.Cron, Timezone: tz, Calendar: cal, Timeout: w.Schedule.Timeout, CatchUp: w.Schedule.CatchUp, DeployLock: deployLock, + RetryAttempts: attempts, RetryBackoff: backoff, RetryMaxBackoff: maxBackoff, + Notify: w.Schedule.notifyOutcomes(), }) } return out, nil diff --git a/internal/app/schedule_retry.go b/internal/app/schedule_retry.go new file mode 100644 index 0000000..c6e69b5 --- /dev/null +++ b/internal/app/schedule_retry.go @@ -0,0 +1,105 @@ +package app + +import "time" + +// The retry defaults reproduce today's behaviour exactly: one attempt, and the +// backoff values are inert until attempts rises above one. +const ( + defaultRetryAttempts = 1 + defaultRetryBackoff = 30 * time.Second + defaultRetryMaxBackoff = 10 * time.Minute + maxRetryAttempts = 10 +) + +// defaultScheduleNotify is what the failure notifier always did: speak on +// failure and timeout, stay quiet on success, and never mention a skip. +var defaultScheduleNotify = []string{"failure", "timeout"} + +// scheduleRetryWorstCase is the longest a run can spend asleep between +// attempts: the doubling series, each term capped, over the attempts-1 sleeps. +// Validation keeps it under the timeout so the last attempt can always start. +func scheduleRetryWorstCase(attempts int, backoff, max time.Duration) time.Duration { + var total time.Duration + sleep := backoff + for i := 1; i < attempts; i++ { + if sleep > max { + sleep = max + } + total += sleep + sleep *= 2 + } + return total +} + +// retryPolicy resolves the declared block over the defaults. Unparseable +// durations fall back to the default here; validation has already refused them +// on the load path, so this only softens a struct built by hand. +func (s *JobSchedule) retryPolicy() (attempts int, backoff, max time.Duration) { + attempts, backoff, max = defaultRetryAttempts, defaultRetryBackoff, defaultRetryMaxBackoff + if s == nil || s.Retry == nil { + return attempts, backoff, max + } + if s.Retry.Attempts != nil { + attempts = *s.Retry.Attempts + } + if s.Retry.Backoff != "" { + if d, ok := ParseDuration(s.Retry.Backoff); ok { + backoff = d + } + } + if s.Retry.MaxBackoff != "" { + if d, ok := ParseDuration(s.Retry.MaxBackoff); ok { + max = d + } + } + return attempts, backoff, max +} + +// notifyOutcomes resolves the declared list over the default. +func (s *JobSchedule) notifyOutcomes() []string { + if s == nil || len(s.Notify) == 0 { + return append([]string(nil), defaultScheduleNotify...) + } + return append([]string(nil), s.Notify...) +} + +// scheduleTimeout is the run's wall-time bound as a duration, with the +// schema default when the author left it out. +func (s *JobSchedule) scheduleTimeout() time.Duration { + if s != nil && s.Timeout != "" { + if d, ok := ParseDuration(s.Timeout); ok { + return d + } + } + return time.Hour +} + +func validateJobRetry(s *JobSchedule, path string) error { + if s.Retry != nil { + if s.Retry.Attempts != nil && (*s.Retry.Attempts < 1 || *s.Retry.Attempts > maxRetryAttempts) { + return errf("project_invalid", path+".retry.attempts", "", + "attempts must be between 1 and %d, got %d", maxRetryAttempts, *s.Retry.Attempts) + } + if err := gDur.checkOptional(path+".retry.backoff", s.Retry.Backoff); err != nil { + return err + } + if err := gDur.checkOptional(path+".retry.max_backoff", s.Retry.MaxBackoff); err != nil { + return err + } + attempts, backoff, max := s.retryPolicy() + if backoff > max { + return errf("project_invalid", path+".retry.backoff", "", + "backoff %s exceeds max_backoff %s", backoff, max) + } + if worst, timeout := scheduleRetryWorstCase(attempts, backoff, max), s.scheduleTimeout(); worst >= timeout { + return errf("project_invalid", path+".retry", "", + "worst-case backoff %s is not smaller than timeout %s; later attempts could never start", worst, timeout) + } + } + for i, outcome := range s.Notify { + if err := checkEnum(indexed(path+".notify", i), outcome, eScheduleNotify); err != nil { + return err + } + } + return nil +} diff --git a/internal/app/schedule_test.go b/internal/app/schedule_test.go index 7236323..4b72b0e 100644 --- a/internal/app/schedule_test.go +++ b/internal/app/schedule_test.go @@ -1,8 +1,10 @@ package app import ( + "errors" "strings" "testing" + "time" ) // A schedule that silently never fires looks exactly like one that works, @@ -148,3 +150,77 @@ workloads: t.Fatal("an invalid scheduled-job timeout was accepted") } } + +func TestScheduledJobRetryAndNotifyResolveWithDefaults(t *testing.T) { + spec, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + plain: + role: job + image: x:1 + data_effect: none + schedule: {cron: "0 3 * * *"} + retrying: + role: job + image: x:1 + data_effect: none + schedule: + cron: "0 * * * *" + timeout: 45m + retry: {attempts: 3, backoff: 30s, max_backoff: 10m} + notify: [failure, timeout, skipped] +`), "ob.yml") + if err != nil { + t.Fatal(err) + } + jobs, err := spec.ScheduledJobs() + if err != nil { + t.Fatal(err) + } + byName := map[string]ScheduledJob{} + for _, job := range jobs { + byName[job.Name] = job + } + plain := byName["plain"] + if plain.RetryAttempts != 1 || plain.RetryBackoff != 30*time.Second || plain.RetryMaxBackoff != 10*time.Minute || + strings.Join(plain.Notify, ",") != "failure,timeout" { + t.Fatalf("defaults did not resolve: %#v", plain) + } + retrying := byName["retrying"] + if retrying.RetryAttempts != 3 || strings.Join(retrying.Notify, ",") != "failure,timeout,skipped" { + t.Fatalf("declared retry did not resolve: %#v", retrying) + } +} + +func TestScheduledJobRetryIsBoundedByTheTimeout(t *testing.T) { + for name, tc := range map[string]struct { + schedule string + code string + }{ + "too many attempts": {`{cron: "0 * * * *", retry: {attempts: 11}}`, "project_invalid"}, + "zero attempts": {`{cron: "0 * * * *", retry: {attempts: 0}}`, "project_invalid"}, + "backoff over max": {`{cron: "0 * * * *", retry: {attempts: 2, backoff: 20m, max_backoff: 10m}}`, "project_invalid"}, + "backoff exceeds timeout": {`{cron: "0 * * * *", timeout: 5m, retry: {attempts: 3, backoff: 2m, max_backoff: 30m}}`, "project_invalid"}, + "unknown notify": {`{cron: "0 * * * *", notify: [warning]}`, "project_invalid"}, + } { + t.Run(name, func(t *testing.T) { + _, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + j: {role: job, image: x:1, data_effect: none, schedule: `+tc.schedule+`} +`), "ob.yml") + var e *Error + if !errors.As(err, &e) || e.Code != tc.code { + t.Fatalf("err = %v, want code %s", err, tc.code) + } + }) + } + if got := scheduleRetryWorstCase(3, 2*time.Minute, 30*time.Minute); got != 6*time.Minute { + t.Fatalf("worst case = %s, want 6m", got) + } + if got := scheduleRetryWorstCase(4, 30*time.Second, time.Minute); got != 150*time.Second { + t.Fatalf("capped worst case = %s, want 2m30s", got) + } +} diff --git a/internal/app/types.go b/internal/app/types.go index 598146f..913051c 100644 --- a/internal/app/types.go +++ b/internal/app/types.go @@ -266,11 +266,22 @@ type Schedule struct { } type JobSchedule struct { - Cron string `json:"cron" description:"Five-field cron schedule translated to a host timer." example:"0 2 * * *"` - Timezone string `json:"timezone" description:"IANA timezone used to interpret the cron schedule." default:"UTC" example:"Europe/Berlin"` - Timeout string `json:"timeout" description:"Maximum wall time for one scheduled run before systemd terminates it and records failure." default:"1h" example:"30m"` - CatchUp bool `json:"catch_up" description:"Run once after the host returns if an elapsed schedule was missed while it was offline." default:"true"` - DeployLock string `json:"deploy_lock" description:"Deployment coordination policy: exclusive blocks application operations for the full run; pinned leases the immutable starting release and permits only deployments without data-changing jobs or untyped hooks." default:"exclusive" example:"pinned"` + Cron string `json:"cron" description:"Five-field cron schedule translated to a host timer." example:"0 2 * * *"` + Timezone string `json:"timezone" description:"IANA timezone used to interpret the cron schedule." default:"UTC" example:"Europe/Berlin"` + Timeout string `json:"timeout" description:"Maximum wall time for one scheduled run before systemd terminates it and records failure." default:"1h" example:"30m"` + CatchUp bool `json:"catch_up" description:"Run once after the host returns if an elapsed schedule was missed while it was offline." default:"true"` + DeployLock string `json:"deploy_lock" description:"Deployment coordination policy: exclusive blocks application operations for the full run; pinned leases the immutable starting release and permits only deployments without data-changing jobs or untyped hooks." default:"exclusive" example:"pinned"` + Retry *JobRetry `json:"retry,omitempty" description:"Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run."` + Notify []string `json:"notify,omitempty" description:"Run outcomes that send the configured notifications: success, failure, timeout, skipped." default:"failure, timeout"` +} + +// JobRetry bounds how a scheduled run recovers from a transient failure. The +// sleeps happen under the locks the run already holds, so validation keeps +// their worst-case sum under the schedule's timeout. +type JobRetry struct { + Attempts *int `json:"attempts,omitempty" description:"Total attempts including the first, 1 to 10." default:"1" example:"3"` + Backoff string `json:"backoff,omitempty" description:"Sleep before the second attempt; it doubles after each failure." default:"30s" example:"1m"` + MaxBackoff string `json:"max_backoff,omitempty" description:"Upper bound for the doubling sleep." default:"10m" example:"30m"` } type Service struct { diff --git a/internal/app/validate.go b/internal/app/validate.go index c1773df..9d58a25 100644 --- a/internal/app/validate.go +++ b/internal/app/validate.go @@ -580,6 +580,9 @@ func validateJobSchedule(s *JobSchedule, path string) error { if err := gDur.check(path+".timeout", s.Timeout); err != nil { return err } + if err := validateJobRetry(s, path); err != nil { + return err + } return checkEnum(path+".deploy_lock", s.DeployLock, eScheduleDeployLock) } diff --git a/site/public/onebox.run-v1.schema.json b/site/public/onebox.run-v1.schema.json index ef54090..9981c67 100644 --- a/site/public/onebox.run-v1.schema.json +++ b/site/public/onebox.run-v1.schema.json @@ -2536,6 +2536,48 @@ ], "type": "string" }, + "notify": { + "default": "failure, timeout", + "description": "Run outcomes that send the configured notifications: success, failure, timeout, skipped.", + "items": { + "type": "string" + }, + "type": "array" + }, + "retry": { + "additionalProperties": false, + "description": "Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run.", + "patternProperties": { + "^x-": {} + }, + "properties": { + "attempts": { + "default": 1, + "description": "Total attempts including the first, 1 to 10.", + "examples": [ + 3 + ], + "type": "integer" + }, + "backoff": { + "default": "30s", + "description": "Sleep before the second attempt; it doubles after each failure.", + "examples": [ + "1m" + ], + "type": "string" + }, + "max_backoff": { + "default": "10m", + "description": "Upper bound for the doubling sleep.", + "examples": [ + "30m" + ], + "type": "string" + } + }, + "type": "object" + }, "timeout": { "default": "1h", "description": "Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d.", diff --git a/site/src/content/docs/reference/fields/workloads.mdx b/site/src/content/docs/reference/fields/workloads.mdx index aa9bcc6..56c5284 100644 --- a/site/src/content/docs/reference/fields/workloads.mdx +++ b/site/src/content/docs/reference/fields/workloads.mdx @@ -19,7 +19,7 @@ cannot drift from what `ob validate` accepts. ## Fields on this page -`args` · `bind` · `build` · `catch_up` · `command` · `compose` · `condition` · `container` · `context` · `cpus` · `cron` · `data_effect` · `deploy_lock` · `dockerfile` · `domain` · `drain` · `driver` · `entrypoint` · `env` · `env_files` · `exec` · `extra_hosts` · `file` · `grace` · `health` · `host` · `hostname` · `http` · `image` · `init` · `interval` · `labels` · `logging` · `memory` · `middlewares` · `mode` · `name` · `needs` · `options` · `path` · `persistence` · `port` · `protocol` · `provider` · `published_ports` · `pull` · `reference` · `replicas` · `resources` · `retries` · `role` · `routes` · `schedule` · `scheme` · `signal` · `source` · `start_period` · `stdin_open` · `strategy` · `target` · `tcp` · `timeout` · `timezone` · `tls` · `tty` · `user` · `volumes` · `wait` · `when` · `within` · `working_dir` +`args` · `attempts` · `backoff` · `bind` · `build` · `catch_up` · `command` · `compose` · `condition` · `container` · `context` · `cpus` · `cron` · `data_effect` · `deploy_lock` · `dockerfile` · `domain` · `drain` · `driver` · `entrypoint` · `env` · `env_files` · `exec` · `extra_hosts` · `file` · `grace` · `health` · `host` · `hostname` · `http` · `image` · `init` · `interval` · `labels` · `logging` · `max_backoff` · `memory` · `middlewares` · `mode` · `name` · `needs` · `notify` · `options` · `path` · `persistence` · `port` · `protocol` · `provider` · `published_ports` · `pull` · `reference` · `replicas` · `resources` · `retries` · `retry` · `role` · `routes` · `schedule` · `scheme` · `signal` · `source` · `start_period` · `stdin_open` · `strategy` · `target` · `tcp` · `timeout` · `timezone` · `tls` · `tty` · `user` · `volumes` · `wait` · `when` · `within` · `working_dir` ## Reference @@ -92,6 +92,11 @@ cannot drift from what `ob validate` accepts. | `.schedule.catch_up` | boolean | `true` | Run once after the host returns if an elapsed schedule was missed while it was offline. | | `.schedule.cron` | string | — | Five-field cron schedule translated to a host timer. Expects five cron fields. | | `.schedule.deploy_lock` | `exclusive` · `pinned` | `exclusive` | Deployment coordination policy: exclusive blocks application operations for the full run; pinned leases the immutable starting release and permits only deployments without data-changing jobs or untyped hooks. | +| `.schedule.notify` | list | `failure, timeout` | Run outcomes that send the configured notifications: success, failure, timeout, skipped. | +| `.schedule.retry` | object | — | Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run. | +| `.schedule.retry.attempts` | integer | `1` | Total attempts including the first, 1 to 10. | +| `.schedule.retry.backoff` | string | `30s` | Sleep before the second attempt; it doubles after each failure. | +| `.schedule.retry.max_backoff` | string | `10m` | Upper bound for the doubling sleep. | | `.schedule.timeout` | string | `1h` | Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d. | | `.schedule.timezone` | string | `UTC` | IANA timezone used to interpret the cron schedule. Expects an IANA zone name such as UTC or Europe/Berlin. | | `.stdin_open` | boolean | — | Keep standard input open for the container. | From 2e4e2638d22efb08bf83d44cdac16d206e77d334 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:25:08 -0700 Subject: [PATCH 08/25] feat(schedule): bounded retry in the runner and per-outcome notifications Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule.go | 174 +++++++++++++++++++++++++------ internal/engine/schedule_test.go | 156 +++++++++++++++++++++++++-- 2 files changed, 287 insertions(+), 43 deletions(-) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 55b0598..d881ee1 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math" "regexp" "strings" @@ -107,7 +108,7 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { runtimeEnvFiles = e.Spec.Runtime.EnvFiles } runner := scheduleRunnerScript(e.Spec.Name, job, n, e.lockPath(), runtimeEnvFiles) - notifier, err := e.scheduleFailureNotifier(job.Name) + notifier, err := e.scheduleNotifier(job) if err != nil { return fmt.Errorf("job %s: cannot render its failure notifier: %w", job.Name, err) } @@ -171,7 +172,7 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { // release before releasing that rendezvous, then retains only its own job lock. func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile) string { if job.DeployLock == "pinned" { - return pinnedScheduleRunnerScript(application, job.Name, names, applicationLock, runtimeEnvFiles) + return pinnedScheduleRunnerScript(application, job, names, applicationLock, runtimeEnvFiles) } container := names.Container(job.Name, 1) projectDir := q(names.CurrentLink()) @@ -200,24 +201,25 @@ func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Na "trap 'exit 143' 15", } lines = append(lines, scheduleRunPreamble(names.ScheduledJobRunState(job.Name))...) - lines = append(lines, "write_state 1", compose, "") + lines = append(lines, scheduleAttemptLoop(job, compose)...) + lines = append(lines, "") return strings.Join(lines, "\n") } -func pinnedScheduleRunnerScript(application, job string, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile) string { +func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile) string { scheduleDir := names.AppDir() + "/schedule" - state := names.ScheduledJobRunState(job) - container := names.Container(job, 1) + state := names.ScheduledJobRunState(job.Name) + container := names.Container(job.Name, 1) projectDir := `"$release_dir"` compose := "/usr/bin/docker compose -p " + q(application) + " --project-directory " + projectDir + " -f " + projectDir + "/" + q("compose.yaml") + scheduleRuntimeEnvArgs(projectDir, runtimeEnvFiles) + - " run --rm --no-deps --name " + q(container) + " " + q(job) + " run --rm --no-deps --name " + q(container) + " " + q(job.Name) lines := []string{ "#!/bin/sh", "# Written by Onebox. Edits are overwritten on the next deploy.", "set -eu", "install -d -m 700 " + q(scheduleDir), - "exec 9>" + q(names.ScheduledJobRunLock(job)), + "exec 9>" + q(names.ScheduledJobRunLock(job.Name)), "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 9", "exec 8>" + q(names.ScheduleRunLock()), "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 8", @@ -238,7 +240,11 @@ func pinnedScheduleRunnerScript(application, job string, names app.Names, applic "trap 'exit 143' 15", } lines = append(lines, scheduleRunPreamble(state)...) - lines = append(lines, "write_state 1", "/usr/bin/flock --unlock 8", compose, "") + // The lease is held; the schedule mutex goes back before the first + // attempt so a compatible deploy is not blocked through the backoff. + lines = append(lines, "/usr/bin/flock --unlock 8") + lines = append(lines, scheduleAttemptLoop(job, compose)...) + lines = append(lines, "") return strings.Join(lines, "\n") } @@ -276,6 +282,34 @@ func scheduleRunPreamble(state string) []string { }, scheduleStateFunction()...) } +// scheduleAttemptLoop runs the container until it exits 0 or the attempts are +// spent. Backoff doubles and is capped; every sleep happens under the locks +// the run already holds, which is why validation keeps the sum under the +// timeout. A single-attempt job gets no loop, so its runner reads as before. +func scheduleAttemptLoop(job app.ScheduledJob, compose string) []string { + if job.RetryAttempts <= 1 { + return []string{"write_state 1", compose} + } + return []string{ + fmt.Sprintf("max_attempts=%d", job.RetryAttempts), + fmt.Sprintf("backoff=%d", int(math.Ceil(job.RetryBackoff.Seconds()))), + fmt.Sprintf("max_backoff=%d", int(math.Ceil(job.RetryMaxBackoff.Seconds()))), + "attempt=1", + "while :; do", + " write_state \"$attempt\"", + " status=0", + " " + compose + " || status=$?", + " [ \"$status\" -eq 0 ] && exit 0", + " if [ \"$attempt\" -ge \"$max_attempts\" ]; then exit \"$status\"; fi", + " echo \"onebox: attempt $attempt of $max_attempts exited $status; retrying in ${backoff}s\" >&2", + " sleep \"$backoff\"", + " backoff=$((backoff * 2))", + " [ \"$backoff\" -gt \"$max_backoff\" ] && backoff=$max_backoff", + " attempt=$((attempt + 1))", + "done", + } +} + func scheduleRuntimeEnvArgs(projectDir string, entries []app.EnvFile) string { args := "" for _, entry := range entries { @@ -364,10 +398,22 @@ func scheduleRunRecordLines(job, state string) []string { } } -// scheduleFailureNotifier extends the existing notification contract to work -// fired directly by systemd. The generated file is mode 0600, keeping webhook +// scheduleNotificationRun marks where the notifier substitutes the run id at +// send time. It travels as the payload's deploy_id: the correlation key an +// operator hands to `ob schedule logs --run`. Nothing else about the run goes +// into a notification; the notify package redacts diagnostics on purpose, and +// attempts, duration and exit status belong to the run record on the host. +const scheduleNotificationRun = "__ONEBOX_SCHEDULE_RUN__" + +// scheduleNotifier extends the existing notification contract to work fired +// directly by systemd. It finalises the run record first, then sends for the +// outcomes the job selected. The generated file is mode 0600, keeping webhook // tokens out of unit metadata, and every send is bounded and fail-open. -func (e *Engine) scheduleFailureNotifier(job string) (string, error) { +// +// Bodies are prepared here, once per outcome class, because the payload +// contract lives in the notify package and the host has no Onebox to ask at +// 2am. Only the timestamp and the run id are filled in on the host. +func (e *Engine) scheduleNotifier(job app.ScheduledJob) (string, error) { environment := e.Opts.Environment if environment == "" { environment = e.Spec.Env @@ -376,46 +422,108 @@ func (e *Engine) scheduleFailureNotifier(job string) (string, error) { "#!/bin/sh", "# Written by Onebox. Edits are overwritten on the next deploy.", "set -u", - "exec 9>" + q(e.names().ScheduledJobRunLock(job)), + "exec 9>" + q(e.names().ScheduledJobRunLock(job.Name)), "if /usr/bin/flock --exclusive --nonblock 9; then", - " " + scheduleContainerCleanup(e.names().Container(job, 1)), + " " + scheduleContainerCleanup(e.names().Container(job.Name, 1)), "fi", } - lines = append(lines, scheduleRunRecordLines(job, e.names().ScheduledJobRunState(job))...) - lines = append(lines, `case "$outcome" in failure|timeout) ;; *) exit 0 ;; esac`) + lines = append(lines, scheduleRunRecordLines(job.Name, e.names().ScheduledJobRunState(job.Name))...) + lines = append(lines, `case " `+strings.Join(job.Notify, " ")+` " in *" $outcome "*) ;; *) exit 0 ;; esac`) + wantsSuccess, wantsFailure := false, false + for _, outcome := range job.Notify { + if outcome == "success" { + wantsSuccess = true + } else { + wantsFailure = true + } + } + var success, failure []string + var err error + if wantsSuccess { + if success, err = e.scheduleNotificationSends(job.Name, environment, "ok"); err != nil { + return "", err + } + } + if wantsFailure { + if failure, err = e.scheduleNotificationSends(job.Name, environment, "fail"); err != nil { + return "", err + } + } + if len(success)+len(failure) > 0 { + lines = append(lines, `ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ')`) + lines = append(lines, `if [ "$outcome" = success ]; then`) + lines = append(lines, orNoop(success)...) + lines = append(lines, "else") + lines = append(lines, orNoop(failure)...) + lines = append(lines, "fi", "wait || true") + } + lines = append(lines, "exit 0", "") + return strings.Join(lines, "\n"), nil +} + +// orNoop keeps a shell branch syntactically present when it has nothing to do. +func orNoop(lines []string) []string { + if len(lines) == 0 { + return []string{" :"} + } + return lines +} + +// scheduleNotificationSends renders one backgrounded curl per notification +// that selects the given status. Status is the notify package's word: ok or +// fail. A failed send is logged and never replaces the job's own result. +func (e *Engine) scheduleNotificationSends(job, environment, status string) ([]string, error) { var sends []string for _, name := range sortedNames(e.Spec.Notifications) { cfg := e.Spec.Notifications[name] - prepared, err := notify.Prepare(cfg, notify.Payload{ + payload := notify.Payload{ App: e.Spec.Name, Env: environment, Host: e.T.Destination(), - Verb: "scheduled job " + job, Status: "fail", - Error: "scheduled job failed; inspect trusted host diagnostics", - TS: scheduleNotificationTimestamp, - }) + Verb: "scheduled job " + job, Status: status, + DeployID: scheduleNotificationRun, TS: scheduleNotificationTimestamp, + } + if status != "ok" { + payload.Error = "scheduled job failed; inspect trusted host diagnostics" + } + prepared, err := notify.Prepare(cfg, payload) if err != nil { - return "", err + return nil, err } if prepared == nil { continue } - body := q(string(prepared.Body)) - if before, after, ok := strings.Cut(string(prepared.Body), scheduleNotificationTimestamp); ok { - body = q(before) + `"$ts"` + q(after) - } curl := "curl --fail --silent --show-error --max-time 5 --request POST" + " --header " + q("Content-Type: "+prepared.ContentType) + " --header " + q("X-Title: "+prepared.Title) + ` --data-binary "$body" ` + q(cfg.Webhook) - sends = append(sends, "(body="+body+"; if ! "+curl+"; then echo "+ + sends = append(sends, " (body="+shellBody(string(prepared.Body))+"; if ! "+curl+"; then echo "+ q("onebox: notification "+name+" failed")+" >&2; fi) &") } - if len(sends) > 0 { - lines = append(lines, `ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ')`) - lines = append(lines, sends...) - lines = append(lines, "wait || true") + return sends, nil +} + +// shellBody quotes a prepared body for the shell, leaving the two runtime +// placeholders as expansions of variables the notifier sets before sending. +func shellBody(body string) string { + var out strings.Builder + for body != "" { + next, placeholder, expansion := -1, "", "" + if i := strings.Index(body, scheduleNotificationTimestamp); i >= 0 { + next, placeholder, expansion = i, scheduleNotificationTimestamp, `"$ts"` + } + if i := strings.Index(body, scheduleNotificationRun); i >= 0 && (next < 0 || i < next) { + next, placeholder, expansion = i, scheduleNotificationRun, `"${INVOCATION_ID:-}"` + } + if next < 0 { + out.WriteString(q(body)) + break + } + if next > 0 { + out.WriteString(q(body[:next])) + } + out.WriteString(expansion) + body = body[next+len(placeholder):] } - lines = append(lines, "exit 0", "") - return strings.Join(lines, "\n"), nil + return out.String() } // calendarExpr is the one string both the host's validator and the installed diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index 14e341d..0217eb3 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -446,7 +446,7 @@ func TestScheduledJobFailureNotifierUsesConfiguredWebhooks(t *testing.T) { } f := &transport.Fake{TargetName: "root@example.internal"} e := New(cfg, testProject(t), f, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) - script, err := e.scheduleFailureNotifier("nightly") + script, err := e.scheduleNotifier(app.ScheduledJob{Name: "nightly", Notify: []string{"failure", "timeout"}}) if err != nil { t.Fatal(err) } @@ -868,7 +868,7 @@ func TestScheduledJobNotifierWritesOneRunRecordToTheJournal(t *testing.T) { cfg := testConfig() f := &transport.Fake{TargetName: "root@example.internal"} e := New(cfg, testProject(t), f, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) - script, err := e.scheduleFailureNotifier("nightly") + script, err := e.scheduleNotifier(app.ScheduledJob{Name: "nightly", Notify: []string{"failure", "timeout"}}) if err != nil { t.Fatal(err) } @@ -898,10 +898,11 @@ func TestScheduledJobNotifierWritesOneRunRecordToTheJournal(t *testing.T) { } } -// runNotifier executes the generated ExecStopPost script with a stub -// systemd-cat, the way systemd would after a run. It returns the record the -// script wrote and whether the state file survived. -func runNotifier(t *testing.T, state string, env map[string]string) (map[string]any, bool) { +// runNotifier executes the generated ExecStopPost script with stub systemd-cat +// and curl binaries, the way systemd would after a run. It returns the record +// the script wrote, whether the state file survived, and every curl +// invocation's arguments, one per element. +func runNotifier(t *testing.T, job app.ScheduledJob, notifications map[string]app.Notification, state string, env map[string]string) (map[string]any, bool, []string) { t.Helper() if runtime.GOOS == "windows" { t.Skip("POSIX shell required") @@ -909,9 +910,9 @@ func runNotifier(t *testing.T, state string, env map[string]string) (map[string] base := t.TempDir() cfg := testConfig() cfg.BasePath = base - cfg.Notifications = nil + cfg.Notifications = notifications e := New(cfg, testProject(t), &transport.Fake{TargetName: "root@example.internal"}, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) - script, err := e.scheduleFailureNotifier("nightly") + script, err := e.scheduleNotifier(job) if err != nil { t.Fatal(err) } @@ -931,6 +932,13 @@ func runNotifier(t *testing.T, state string, env map[string]string) (map[string] if err := os.WriteFile(filepath.Join(bin, "systemd-cat"), []byte(stub), 0o755); err != nil { t.Fatal(err) } + sent := filepath.Join(bin, "curl.args") + // Sends run in the background concurrently, so each stub call writes its + // whole argument list in one printf, keeping calls from interleaving. + curl := "#!/bin/sh\nprintf '%s\\n' \"$@\" -- >>" + sent + "\n" + if err := os.WriteFile(filepath.Join(bin, "curl"), []byte(curl), 0o755); err != nil { + t.Fatal(err) + } command := exec.CommandContext(context.Background(), "sh", "-s") command.Stdin = strings.NewReader(script) command.Env = append([]string{"PATH=" + bin + ":" + os.Getenv("PATH")}, "HOME="+base) @@ -953,7 +961,11 @@ func runNotifier(t *testing.T, state string, env map[string]string) (map[string] t.Fatalf("record is not JSON: %v\n%s", err, lines[0]) } _, stateErr := os.Stat(statePath) - return decoded, stateErr == nil + var sends []string + if args, err := os.ReadFile(sent); err == nil { + sends = strings.Split(strings.TrimSpace(string(args)), "\n") + } + return decoded, stateErr == nil, sends } func TestScheduledJobNotifierRecordsEachOutcomeAndRemovesState(t *testing.T) { @@ -972,7 +984,10 @@ func TestScheduledJobNotifierRecordsEachOutcomeAndRemovesState(t *testing.T) { "no state": {"", map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "3"}, "failure", float64(3), 0}, } { t.Run(name, func(t *testing.T) { - record, stateLeft := runNotifier(t, tc.state, tc.env) + record, stateLeft, sends := runNotifier(t, app.ScheduledJob{Name: "nightly", Notify: []string{"failure", "timeout"}}, nil, tc.state, tc.env) + if len(sends) != 0 { + t.Fatalf("no webhook is configured, yet curl ran: %v", sends) + } if record["outcome"] != tc.outcome || record["exit_status"] != tc.exit || record["attempts"] != tc.attempt { t.Fatalf("record = %#v", record) } @@ -1079,3 +1094,124 @@ ActiveState=active t.Fatalf("issue does not name the outcome: %#v", got.Issues) } } + +func TestScheduledJobRunnerRetriesWithCappedDoublingBackoff(t *testing.T) { + job := app.ScheduledJob{Name: "nightly", Timeout: "45m", DeployLock: "exclusive", + RetryAttempts: 3, RetryBackoff: 30 * time.Second, RetryMaxBackoff: 10 * time.Minute} + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil) + for _, want := range []string{ + "max_attempts=3", "backoff=30", "max_backoff=600", + "attempt=1", "while :; do", "write_state \"$attempt\"", + "status=0", "|| status=$?", "[ \"$status\" -eq 0 ] && exit 0", + "if [ \"$attempt\" -ge \"$max_attempts\" ]; then exit \"$status\"; fi", + "sleep \"$backoff\"", "backoff=$((backoff * 2))", "attempt=$((attempt + 1))", + } { + if !strings.Contains(runner, want) { + t.Errorf("runner is missing %q:\n%s", want, runner) + } + } + single := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil) + if strings.Contains(single, "while :; do") { + t.Errorf("a single-attempt job must not carry a retry loop:\n%s", single) + } + pinned := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "pinned", RetryAttempts: 2, RetryBackoff: time.Second, RetryMaxBackoff: time.Minute}, names, "/var/lib/ob/sample/lock", nil) + if !strings.Contains(pinned, "while :; do") || strings.Index(pinned, "flock --unlock 8") > strings.Index(pinned, "while :; do") { + t.Errorf("pinned runner must release the schedule mutex before its attempt loop:\n%s", pinned) + } + for _, script := range []string{runner, single, pinned} { + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(script) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("runner is not valid POSIX shell: %v: %s\n%s", err, output, script) + } + } +} + +func TestScheduledJobNotifierSelectsBodiesByOutcome(t *testing.T) { + cfg := testConfig() + cfg.Notifications = map[string]app.Notification{ + "ops": {Webhook: "https://hooks.example.com/ops", On: []string{"success", "failure"}, Format: "json"}, + } + f := &transport.Fake{TargetName: "root@example.internal"} + e := New(cfg, testProject(t), f, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + script, err := e.scheduleNotifier(app.ScheduledJob{Name: "nightly", Notify: []string{"success", "failure", "skipped"}}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `case " success failure skipped " in *" $outcome "*) ;; *) exit 0 ;; esac`, + `"status":"ok"`, + `"status":"fail"`, + `if [ "$outcome" = success ]; then`, + `"deploy_id":"'"${INVOCATION_ID:-}"'"`, + } { + if !strings.Contains(script, want) { + t.Errorf("notifier is missing %q:\n%s", want, script) + } + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(script) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("notifier is not valid POSIX shell: %v: %s\n%s", err, output, script) + } +} + +func TestScheduledJobNotifierSendsOnlySelectedOutcomesWithTheRunID(t *testing.T) { + state := "release=r1\nstarted_at=2026-09-05T15:00:01Z\nstarted_epoch=1\ntrigger=timer\noperation=\nattempt=2\ninputs=\n" + webhooks := map[string]app.Notification{ + "ops": {Webhook: "https://hooks.example.com/ops", On: []string{"success", "failure"}, Format: "json"}, + "chat": {Webhook: "https://hooks.example.com/chat", On: []string{"failure"}, Format: "text"}, + } + for name, tc := range map[string]struct { + notify []string + env map[string]string + want int + status string + }{ + "failure selected": {[]string{"failure", "timeout"}, map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "1", "INVOCATION_ID": "abc123"}, 2, "fail"}, + "success not selected": {[]string{"failure", "timeout"}, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "abc123"}, 0, ""}, + "success selected": {[]string{"success"}, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "abc123"}, 1, "ok"}, + "skipped selected": {[]string{"skipped"}, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "75", "INVOCATION_ID": "abc123"}, 2, "fail"}, + "timeout not selected": {[]string{"failure"}, map[string]string{"SERVICE_RESULT": "timeout", "EXIT_STATUS": "TERM", "INVOCATION_ID": "abc123"}, 0, ""}, + } { + t.Run(name, func(t *testing.T) { + _, _, sends := runNotifier(t, app.ScheduledJob{Name: "nightly", Notify: tc.notify}, webhooks, state, tc.env) + calls := 0 + for _, arg := range sends { + if arg == "--" { + calls++ + } + } + if calls != tc.want { + t.Fatalf("curl ran %d time(s), want %d:\n%s", calls, tc.want, strings.Join(sends, "\n")) + } + if tc.want == 0 { + return + } + joined := strings.Join(sends, "\n") + var jsonBody string + for _, arg := range sends { + if strings.HasPrefix(arg, "{") { + jsonBody = arg + } + } + if jsonBody == "" { + t.Fatalf("no JSON body was sent:\n%s", joined) + } + var body map[string]any + if err := json.Unmarshal([]byte(jsonBody), &body); err != nil { + t.Fatalf("sent body is not JSON: %v\n%s", err, jsonBody) + } + if body["status"] != tc.status || body["deploy_id"] != "abc123" || body["verb"] != "scheduled job nightly" { + t.Fatalf("body = %#v", body) + } + if ts, _ := body["ts"].(string); !strings.HasSuffix(ts, "Z") || strings.Contains(ts, "ONEBOX") { + t.Fatalf("timestamp was not filled at send time: %#v", body) + } + if strings.Contains(joined, "ONEBOX_SCHEDULE") { + t.Fatalf("a placeholder leaked into a send:\n%s", joined) + } + }) + } +} From 6d7f3751b9acd5712e651203ef60cc6950dfb37f Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:27:55 -0700 Subject: [PATCH 09/25] feat(schema): declared inputs for scheduled jobs, rendered as environment defaults Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- docs/onebox.run-v1.schema.json | 38 ++++++ internal/app/generate.go | 14 +- internal/app/names.go | 6 + internal/app/schedule.go | 4 +- internal/app/schedule_inputs.go | 123 +++++++++++++++++ internal/app/schedule_test.go | 124 ++++++++++++++++++ internal/app/types.go | 18 ++- internal/app/validate.go | 7 +- site/public/onebox.run-v1.schema.json | 38 ++++++ .../docs/reference/fields/workloads.mdx | 7 +- 10 files changed, 371 insertions(+), 8 deletions(-) create mode 100644 internal/app/schedule_inputs.go diff --git a/docs/onebox.run-v1.schema.json b/docs/onebox.run-v1.schema.json index 9981c67..4344bd9 100644 --- a/docs/onebox.run-v1.schema.json +++ b/docs/onebox.run-v1.schema.json @@ -2231,6 +2231,44 @@ "description": "Run a minimal init process as PID 1 inside the container.", "type": "boolean" }, + "inputs": { + "additionalProperties": { + "additionalProperties": false, + "patternProperties": { + "^x-": {} + }, + "properties": { + "default": { + "description": "Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint.", + "type": "string" + }, + "description": { + "description": "What the input controls.", + "type": "string" + }, + "enum": { + "description": "Accepted values.", + "examples": [ + "catalog" + ], + "items": { + "type": "string" + }, + "type": "array" + }, + "pattern": { + "description": "Regular expression the whole value must match.", + "examples": [ + "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + ], + "type": "string" + } + }, + "type": "object" + }, + "description": "Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them.", + "type": "object" + }, "labels": { "additionalProperties": {}, "description": "Additional container labels outside namespaces reserved by Onebox and the proxy.", diff --git a/internal/app/generate.go b/internal/app/generate.go index 2087c78..1ba9e7a 100644 --- a/internal/app/generate.go +++ b/internal/app/generate.go @@ -344,7 +344,19 @@ func (p *Spec) renderWorkload(n Names, name string, w Workload, releaseID string } svc["labels"] = labels - if env := stringMap(w.Env); len(env) > 0 { + env := stringMap(w.Env) + // A declared input's default is part of the release, so a timer firing, a + // manual run without overrides, and a hand-typed `docker compose run` all + // see the same value. Validation refuses a name that is also an env key. + if len(w.Inputs) > 0 { + if env == nil { + env = map[string]any{} + } + for name, in := range w.Inputs { + env[name] = in.Default + } + } + if len(env) > 0 { svc["environment"] = env } // A workload that needs a service reads how to reach it. The file is diff --git a/internal/app/names.go b/internal/app/names.go index 5f36a64..6891c12 100644 --- a/internal/app/names.go +++ b/internal/app/names.go @@ -244,6 +244,12 @@ func (n Names) ScheduledJobRunState(job string) string { return path.Join(n.AppDir(), "schedule", job+".state") } +// ScheduledJobRunInputs is the one-shot file `ob schedule run` leaves for the +// next manual activation. The runner consumes and deletes it. +func (n Names) ScheduledJobRunInputs(job string) string { + return path.Join(n.AppDir(), "schedule", job+".inputs") +} + // ScheduledJobUnitPrefixes returns the current namespace followed by the // pre-2026.8.6 spelling when the application name contains a hyphen. func (n Names) ScheduledJobUnitPrefixes() []string { diff --git a/internal/app/schedule.go b/internal/app/schedule.go index fddeedb..f74e0d9 100644 --- a/internal/app/schedule.go +++ b/internal/app/schedule.go @@ -39,6 +39,8 @@ type ScheduledJob struct { RetryBackoff time.Duration RetryMaxBackoff time.Duration Notify []string + // Inputs are the declared parameters a manual run may override. + Inputs map[string]JobInput } // ScheduledJobs lists every job with a schedule, in a stable order. @@ -67,7 +69,7 @@ func (p *Spec) ScheduledJobs() ([]ScheduledJob, error) { Name: name, Cron: w.Schedule.Cron, Timezone: tz, Calendar: cal, Timeout: w.Schedule.Timeout, CatchUp: w.Schedule.CatchUp, DeployLock: deployLock, RetryAttempts: attempts, RetryBackoff: backoff, RetryMaxBackoff: maxBackoff, - Notify: w.Schedule.notifyOutcomes(), + Notify: w.Schedule.notifyOutcomes(), Inputs: w.Inputs, }) } return out, nil diff --git a/internal/app/schedule_inputs.go b/internal/app/schedule_inputs.go new file mode 100644 index 0000000..90bb0c3 --- /dev/null +++ b/internal/app/schedule_inputs.go @@ -0,0 +1,123 @@ +package app + +import ( + "fmt" + "regexp" + "strings" +) + +// ReservedInputPrefix is Onebox's environment namespace. Declared inputs may +// not enter it, and the manual-run metadata line the runner reads lives there. +const ReservedInputPrefix = "ONEBOX_" + +// maxInputValueBytes bounds a value the way the exec reason is bounded: it is +// public operational metadata that travels through a file, a command line and +// a journal record, and none of those want a paragraph. +const maxInputValueBytes = 256 + +// gInputName is deliberately narrower than gEnvName: inputs become +// environment variables, and the upper-case convention keeps them visibly +// distinct from whatever the image sets for itself. +var gInputName = grammar{"input name", regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`), + "upper-case letters, digits and underscores, starting with a letter"} + +// InputValueAllowed is the charset that lets the runner pass a value through +// to the container and into the run record without escaping anything. It is +// checked regardless of the declared pattern, because a pattern is the +// project's promise about meaning, not the runner's guarantee about shell. +func InputValueAllowed(v string) bool { + if len(v) > maxInputValueBytes { + return false + } + for _, r := range v { + if r == '"' || r == '\\' || r < 0x20 || r == 0x7f { + return false + } + } + return true +} + +// accepts reports whether v satisfies this input. A pattern matches the whole +// value: an author who writes `[0-9]+` means a number, not a value that +// contains one somewhere. +func (in JobInput) accepts(v string) bool { + if !InputValueAllowed(v) { + return false + } + if len(in.Enum) > 0 { + for _, allowed := range in.Enum { + if v == allowed { + return true + } + } + return false + } + re, err := regexp.Compile("^(?:" + in.Pattern + ")$") + if err != nil { + return false + } + return re.MatchString(v) +} + +// ValidateJobInputValues checks operator-supplied overrides against the +// declaration. Defaults were checked at load; this is the other half, run on +// the workstation before anything reaches the host. +func ValidateJobInputValues(w Workload, values map[string]string) error { + for _, name := range sortedKeys(values) { + in, ok := w.Inputs[name] + if !ok { + return fmt.Errorf("input %s is not declared by this job", name) + } + if !in.accepts(values[name]) { + return fmt.Errorf("input %s: %q is not an accepted value", name, values[name]) + } + } + return nil +} + +func validateJobInputs(w Workload, path string) error { + if len(w.Inputs) == 0 { + return nil + } + if w.Schedule == nil { + return errf("project_invalid", path+".inputs", "", + "inputs belong to a scheduled job; declare schedule or remove inputs") + } + if w.DataEffect != DataEffectNone { + return errf("project_invalid", path+".inputs", "", + "inputs require data_effect %q; a %q job keeps the sealed plan of ob job run for operator-initiated runs", + DataEffectNone, w.DataEffect) + } + for _, name := range sortedKeys(w.Inputs) { + ip := path + ".inputs." + name + if err := gInputName.check(ip, name); err != nil { + return err + } + if strings.HasPrefix(name, ReservedInputPrefix) { + return errf("project_invalid", ip, "", "%s is Onebox's environment namespace; choose another name", ReservedInputPrefix) + } + if _, clash := w.Env[name]; clash { + return errf("project_invalid", ip, "", "input %s collides with an env key of the same name", name) + } + in := w.Inputs[name] + if (len(in.Enum) > 0) == (in.Pattern != "") { + return errf("project_invalid", ip, "", "declare exactly one of enum or pattern") + } + if in.Pattern != "" { + if _, err := regexp.Compile("^(?:" + in.Pattern + ")$"); err != nil { + return errf("project_invalid", ip+".pattern", "", "%v", err) + } + } + for i, v := range in.Enum { + if !InputValueAllowed(v) { + return errf("project_invalid", indexed(ip+".enum", i), "", + "enum values may not contain quotes, backslashes or control characters and are at most %d bytes", maxInputValueBytes) + } + } + if !in.accepts(in.Default) { + return errf("project_invalid", ip+".default", "", + "default %q does not satisfy the input's own constraint", in.Default) + } + } + return nil +} diff --git a/internal/app/schedule_test.go b/internal/app/schedule_test.go index 4b72b0e..3dc250f 100644 --- a/internal/app/schedule_test.go +++ b/internal/app/schedule_test.go @@ -2,6 +2,7 @@ package app import ( "errors" + "fmt" "strings" "testing" "time" @@ -224,3 +225,126 @@ workloads: t.Fatalf("capped worst case = %s, want 2m30s", got) } } + +func TestJobInputsValidateNamesConstraintsAndDefaults(t *testing.T) { + base := `api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + sync: + role: job + image: x:1 + data_effect: %s + env: {MODE: fast} + %s + inputs: + %s +` + load := func(effect, schedule, inputs string) error { + _, err := LoadBytes([]byte(fmt.Sprintf(base, effect, schedule, inputs)), "ob.yml") + return err + } + good := "SOURCE: {enum: [catalog, prices], default: catalog, description: Which upstream.}" + if err := load("none", `schedule: {cron: "0 * * * *"}`, good); err != nil { + t.Fatalf("valid inputs refused: %v", err) + } + for name, tc := range map[string]struct{ effect, schedule, inputs string }{ + "no schedule": {"none", "", good}, + "destructive job": {"destructive", `schedule: {cron: "0 * * * *"}`, good}, + "lowercase name": {"none", `schedule: {cron: "0 * * * *"}`, "source: {enum: [a], default: a}"}, + "reserved prefix": {"none", `schedule: {cron: "0 * * * *"}`, "ONEBOX_X: {enum: [a], default: a}"}, + "collides with env": {"none", `schedule: {cron: "0 * * * *"}`, "MODE: {enum: [a], default: a}"}, + "enum and pattern": {"none", `schedule: {cron: "0 * * * *"}`, "S: {enum: [a], pattern: '^a$', default: a}"}, + "neither": {"none", `schedule: {cron: "0 * * * *"}`, "S: {default: a}"}, + "default off enum": {"none", `schedule: {cron: "0 * * * *"}`, "S: {enum: [a], default: b}"}, + "default off pattern": {"none", `schedule: {cron: "0 * * * *"}`, "S: {pattern: '^[0-9]+$', default: x}"}, + "quote in default": {"none", `schedule: {cron: "0 * * * *"}`, `S: {pattern: '.*', default: 'a"b'}`}, + "bad regex": {"none", `schedule: {cron: "0 * * * *"}`, "S: {pattern: '(', default: a}"}, + } { + t.Run(name, func(t *testing.T) { + var e *Error + if err := load(tc.effect, tc.schedule, tc.inputs); !errors.As(err, &e) || e.Code != "project_invalid" { + t.Fatalf("err = %v, want project_invalid", err) + } + }) + } + if _, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + web: {role: application, image: x:1, inputs: {S: {enum: [a], default: a}}} +`), "ob.yml"); err == nil { + t.Fatal("inputs on a non-job workload were accepted") + } +} + +func TestValidateJobInputValuesChecksOverrides(t *testing.T) { + w := Workload{Role: RoleJob, Inputs: map[string]JobInput{ + "SOURCE": {Enum: []string{"catalog", "prices"}, Default: "catalog"}, + "SINCE": {Pattern: `^([0-9]{4}-[0-9]{2}-[0-9]{2})?$`, Default: ""}, + }} + if err := ValidateJobInputValues(w, map[string]string{"SOURCE": "prices", "SINCE": "2026-09-01"}); err != nil { + t.Fatal(err) + } + if err := ValidateJobInputValues(w, nil); err != nil { + t.Fatal(err) + } + for name, values := range map[string]map[string]string{ + "unknown": {"OTHER": "x"}, + "off enum": {"SOURCE": "reviews"}, + "off pattern": {"SINCE": "yesterday"}, + "backslash": {"SINCE": `2026\-09-01`}, + "newline": {"SOURCE": "prices\n"}, + "partial match": {"SINCE": "x2026-09-01"}, + } { + if err := ValidateJobInputValues(w, values); err == nil { + t.Errorf("%s was accepted", name) + } + } + if InputValueAllowed(strings.Repeat("a", 257)) { + t.Error("an oversized value was allowed") + } + if !InputValueAllowed("a value with spaces, commas, and unicode ✓") { + t.Error("an ordinary value was refused") + } +} + +func TestScheduledJobInputDefaultsRenderIntoTheComposeEnvironment(t *testing.T) { + spec, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + sync: + role: job + image: x:1 + data_effect: none + env: {MODE: fast} + schedule: {cron: "0 * * * *"} + inputs: + SOURCE: {enum: [catalog, prices], default: catalog} +`), "ob.yml") + if err != nil { + t.Fatal(err) + } + r, err := spec.Resolve("production") + if err != nil { + t.Fatal(err) + } + out, err := r.Render("production", "R1", nil) + if err != nil { + t.Fatal(err) + } + rendered := string(out.Bytes) + for _, want := range []string{"SOURCE: catalog", "MODE: fast"} { + if !strings.Contains(rendered, want) { + t.Fatalf("rendered runtime is missing %q:\n%s", want, rendered) + } + } + jobs, err := spec.ScheduledJobs() + if err != nil { + t.Fatal(err) + } + if jobs[0].Inputs["SOURCE"].Default != "catalog" { + t.Fatalf("scheduled job did not carry its inputs: %#v", jobs[0]) + } +} diff --git a/internal/app/types.go b/internal/app/types.go index 913051c..841a324 100644 --- a/internal/app/types.go +++ b/internal/app/types.go @@ -168,9 +168,21 @@ type Workload struct { Logging *Logging `json:"logging,omitempty" description:"Container logging driver and driver-specific options."` // Job only. - When string `json:"when,omitempty" description:"When a job runs: manual, pre_release, or post_release." default:"manual"` - DataEffect DataEffect `json:"data_effect,omitempty" description:"Job data impact used by rollback and abort gates." example:"migration"` - Schedule *JobSchedule `json:"schedule,omitempty" description:"Host-resident recurring schedule and run policy for a job."` + When string `json:"when,omitempty" description:"When a job runs: manual, pre_release, or post_release." default:"manual"` + DataEffect DataEffect `json:"data_effect,omitempty" description:"Job data impact used by rollback and abort gates." example:"migration"` + Schedule *JobSchedule `json:"schedule,omitempty" description:"Host-resident recurring schedule and run policy for a job."` + Inputs map[string]JobInput `json:"inputs,omitempty" description:"Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them."` +} + +// JobInput is one declared parameter of a scheduled job. The constraint is +// what makes a manual run safe to accept from a command line: a value is +// either one of the listed words or matches the pattern, and never contains a +// character the runner would have to escape. +type JobInput struct { + Enum []string `json:"enum,omitempty" description:"Accepted values." example:"catalog"` + Pattern string `json:"pattern,omitempty" description:"Regular expression the whole value must match." example:"^[0-9]{4}-[0-9]{2}-[0-9]{2}$"` + Default string `json:"default" description:"Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint."` + Description string `json:"description,omitempty" description:"What the input controls."` } type Build struct { diff --git a/internal/app/validate.go b/internal/app/validate.go index 9d58a25..3af3d6f 100644 --- a/internal/app/validate.go +++ b/internal/app/validate.go @@ -455,9 +455,12 @@ func validateWorkload(w Workload, path string) error { "pinned scheduled runs require a Onebox-rendered workload; adopted Compose may reference files outside the leased release") } } - } else if w.When != "" || w.DataEffect != "" || w.Schedule != nil { + if err := validateJobInputs(w, path); err != nil { + return err + } + } else if w.When != "" || w.DataEffect != "" || w.Schedule != nil || len(w.Inputs) > 0 { return errf("project_invalid", path, "", - "when, data_effect and schedule belong to a job; this workload's role is %q", w.Role) + "when, data_effect, schedule and inputs belong to a job; this workload's role is %q", w.Role) } return nil } diff --git a/site/public/onebox.run-v1.schema.json b/site/public/onebox.run-v1.schema.json index 9981c67..4344bd9 100644 --- a/site/public/onebox.run-v1.schema.json +++ b/site/public/onebox.run-v1.schema.json @@ -2231,6 +2231,44 @@ "description": "Run a minimal init process as PID 1 inside the container.", "type": "boolean" }, + "inputs": { + "additionalProperties": { + "additionalProperties": false, + "patternProperties": { + "^x-": {} + }, + "properties": { + "default": { + "description": "Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint.", + "type": "string" + }, + "description": { + "description": "What the input controls.", + "type": "string" + }, + "enum": { + "description": "Accepted values.", + "examples": [ + "catalog" + ], + "items": { + "type": "string" + }, + "type": "array" + }, + "pattern": { + "description": "Regular expression the whole value must match.", + "examples": [ + "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + ], + "type": "string" + } + }, + "type": "object" + }, + "description": "Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them.", + "type": "object" + }, "labels": { "additionalProperties": {}, "description": "Additional container labels outside namespaces reserved by Onebox and the proxy.", diff --git a/site/src/content/docs/reference/fields/workloads.mdx b/site/src/content/docs/reference/fields/workloads.mdx index 56c5284..044559f 100644 --- a/site/src/content/docs/reference/fields/workloads.mdx +++ b/site/src/content/docs/reference/fields/workloads.mdx @@ -19,7 +19,7 @@ cannot drift from what `ob validate` accepts. ## Fields on this page -`args` · `attempts` · `backoff` · `bind` · `build` · `catch_up` · `command` · `compose` · `condition` · `container` · `context` · `cpus` · `cron` · `data_effect` · `deploy_lock` · `dockerfile` · `domain` · `drain` · `driver` · `entrypoint` · `env` · `env_files` · `exec` · `extra_hosts` · `file` · `grace` · `health` · `host` · `hostname` · `http` · `image` · `init` · `interval` · `labels` · `logging` · `max_backoff` · `memory` · `middlewares` · `mode` · `name` · `needs` · `notify` · `options` · `path` · `persistence` · `port` · `protocol` · `provider` · `published_ports` · `pull` · `reference` · `replicas` · `resources` · `retries` · `retry` · `role` · `routes` · `schedule` · `scheme` · `signal` · `source` · `start_period` · `stdin_open` · `strategy` · `target` · `tcp` · `timeout` · `timezone` · `tls` · `tty` · `user` · `volumes` · `wait` · `when` · `within` · `working_dir` +`args` · `attempts` · `backoff` · `bind` · `build` · `catch_up` · `command` · `compose` · `condition` · `container` · `context` · `cpus` · `cron` · `data_effect` · `default` · `deploy_lock` · `description` · `dockerfile` · `domain` · `drain` · `driver` · `entrypoint` · `enum` · `env` · `env_files` · `exec` · `extra_hosts` · `file` · `grace` · `health` · `host` · `hostname` · `http` · `image` · `init` · `inputs` · `interval` · `labels` · `logging` · `max_backoff` · `memory` · `middlewares` · `mode` · `name` · `needs` · `notify` · `options` · `path` · `pattern` · `persistence` · `port` · `protocol` · `provider` · `published_ports` · `pull` · `reference` · `replicas` · `resources` · `retries` · `retry` · `role` · `routes` · `schedule` · `scheme` · `signal` · `source` · `start_period` · `stdin_open` · `strategy` · `target` · `tcp` · `timeout` · `timezone` · `tls` · `tty` · `user` · `volumes` · `wait` · `when` · `within` · `working_dir` ## Reference @@ -58,6 +58,11 @@ cannot drift from what `ob validate` accepts. | `.image.pull` | `always` · `missing` · `never` | `missing` | When to fetch the image from the registry: missing fetches only what the host does not already hold, always fetches every release, never fetches at all and fails on a missing image. | | `.image.reference` | string | — | Complete container image reference, optionally tagged or digest-pinned. Expects a registry reference such as nginx:1.27 or ghcr.io/acme/app@sha256:…. | | `.init` | boolean | — | Run a minimal init process as PID 1 inside the container. | +| `.inputs` | map | — | Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them. | +| `.inputs..default` | string | — | Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint. | +| `.inputs..description` | string | — | What the input controls. | +| `.inputs..enum` | list | — | Accepted values. | +| `.inputs..pattern` | string | — | Regular expression the whole value must match. | | `.labels` | map | — | Additional container labels outside namespaces reserved by Onebox and the proxy. | | `.logging` | object | — | Container logging driver and driver-specific options. | | `.logging.driver` | string | — | Container runtime logging driver. Expects a log driver name such as local, json-file or an org/plugin:tag. | From 39f88803ad894373bd12a8bd800e55688a40c870 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:30:17 -0700 Subject: [PATCH 10/25] feat(schedule): manual activations consume declared inputs; hosts need systemd 252 for them Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule.go | 92 +++++++++++++++++---- internal/engine/schedule_test.go | 134 ++++++++++++++++++++++++++++++- 2 files changed, 208 insertions(+), 18 deletions(-) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index d881ee1..d39e081 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "regexp" + "strconv" "strings" "github.com/labstack/onebox/internal/app" @@ -83,6 +84,20 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { if len(jobs) > 0 && !e.hasFlock(ctx) { return errors.New("scheduled jobs require flock on the target so they cannot overlap deployments; install util-linux and deploy again") } + if needsTriggerUnit(jobs) { + // A manual activation is told apart from a timer firing by the + // TRIGGER_UNIT variable systemd 252 introduced. Without it the runner + // could not know whether to read the inputs file, so inputs stay + // refused on an older host rather than guessed at. + res, err := e.T.Run(ctx, "systemctl --version 2>/dev/null | head -1") + if err != nil { + return err + } + if version, ok := systemdVersion(res.Stdout); !ok || version < 252 { + return fmt.Errorf("a job declares inputs, which need systemd 252 or newer on the host for $TRIGGER_UNIT; the host reports %q", + strings.TrimSpace(res.Stdout)) + } + } for _, job := range jobs { unit := n.ScheduledJobUnit(job.Name) wanted[unit] = true @@ -178,28 +193,31 @@ func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Na projectDir := q(names.CurrentLink()) compose := "/usr/bin/docker compose -p " + q(application) + " --project-directory " + projectDir + " -f " + projectDir + "/" + q("compose.yaml") + scheduleRuntimeEnvArgs(projectDir, runtimeEnvFiles) + - " run --rm --no-deps --name " + q(container) + " " + q(job.Name) + " run --rm --no-deps \"$@\" --name " + q(container) + " " + q(job.Name) lines := []string{ "#!/bin/sh", "# Written by Onebox. Edits are overwritten on the next deploy.", "set -eu", "install -d -m 700 " + q(names.AppDir()+"/schedule"), - "exec 9>" + q(names.ScheduledJobRunLock(job.Name)), + } + lines = append(lines, scheduleInputsLines(names.ScheduledJobRunInputs(job.Name))...) + lines = append(lines, + "exec 9>"+q(names.ScheduledJobRunLock(job.Name)), "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 9", - "exec 8>" + q(names.ScheduleRunLock()), + "exec 8>"+q(names.ScheduleRunLock()), "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 8", - "if [ -e " + q(applicationLock) + " ]; then echo 'onebox: an application operation holds the deploy lock' >&2; exit 75; fi", + "if [ -e "+q(applicationLock)+" ]; then echo 'onebox: an application operation holds the deploy lock' >&2; exit 75; fi", // Best effort: the record names the release that ran, and an exclusive // job runs whatever `current` points at when it starts. - "release_dir=$(readlink -f " + q(names.CurrentLink()) + " 2>/dev/null || true)", + "release_dir=$(readlink -f "+q(names.CurrentLink())+" 2>/dev/null || true)", "release=${release_dir##*/}", scheduleContainerCleanup(container), - "cleanup() { " + scheduleContainerCleanup(container) + "; rm -f \"$tmp\"; }", + "cleanup() { "+scheduleContainerCleanup(container)+"; rm -f \"$tmp\"; }", "trap cleanup 0", "trap 'exit 129' 1", "trap 'exit 130' 2", "trap 'exit 143' 15", - } + ) lines = append(lines, scheduleRunPreamble(names.ScheduledJobRunState(job.Name))...) lines = append(lines, scheduleAttemptLoop(job, compose)...) lines = append(lines, "") @@ -213,19 +231,22 @@ func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names projectDir := `"$release_dir"` compose := "/usr/bin/docker compose -p " + q(application) + " --project-directory " + projectDir + " -f " + projectDir + "/" + q("compose.yaml") + scheduleRuntimeEnvArgs(projectDir, runtimeEnvFiles) + - " run --rm --no-deps --name " + q(container) + " " + q(job.Name) + " run --rm --no-deps \"$@\" --name " + q(container) + " " + q(job.Name) lines := []string{ "#!/bin/sh", "# Written by Onebox. Edits are overwritten on the next deploy.", "set -eu", "install -d -m 700 " + q(scheduleDir), - "exec 9>" + q(names.ScheduledJobRunLock(job.Name)), + } + lines = append(lines, scheduleInputsLines(names.ScheduledJobRunInputs(job.Name))...) + lines = append(lines, + "exec 9>"+q(names.ScheduledJobRunLock(job.Name)), "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 9", - "exec 8>" + q(names.ScheduleRunLock()), + "exec 8>"+q(names.ScheduleRunLock()), "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 8", - "if [ -e " + q(applicationLock) + " ]; then echo 'onebox: an application operation holds the deploy lock' >&2; exit 75; fi", - "release_dir=$(readlink -f " + q(names.CurrentLink()) + ") || { echo 'onebox: current release cannot be resolved' >&2; exit 75; }", - "if [ \"${release_dir%/*}\" != " + q(names.ReleasesDir()) + " ]; then echo 'onebox: current release resolves outside the release store' >&2; exit 75; fi", + "if [ -e "+q(applicationLock)+" ]; then echo 'onebox: an application operation holds the deploy lock' >&2; exit 75; fi", + "release_dir=$(readlink -f "+q(names.CurrentLink())+") || { echo 'onebox: current release cannot be resolved' >&2; exit 75; }", + "if [ \"${release_dir%/*}\" != "+q(names.ReleasesDir())+" ]; then echo 'onebox: current release resolves outside the release store' >&2; exit 75; fi", "release=${release_dir##*/}", "if ! printf '%s\\n' \"$release\" | grep -Eq '^[0-9]{8}-[0-9]{6}-[0-9A-Za-z_-]+$'; then echo 'onebox: current release identity is invalid' >&2; exit 75; fi", "if [ ! -f \"$release_dir/compose.yaml\" ]; then echo 'onebox: pinned release has no compose.yaml' >&2; exit 75; fi", @@ -233,12 +254,12 @@ func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names "chmod 600 \"$release_dir/.ob-schedule.lease\"", "/usr/bin/flock --shared 7", scheduleContainerCleanup(container), - "cleanup() { " + scheduleContainerCleanup(container) + "; rm -f \"$tmp\"; }", + "cleanup() { "+scheduleContainerCleanup(container)+"; rm -f \"$tmp\"; }", "trap cleanup 0", "trap 'exit 129' 1", "trap 'exit 130' 2", "trap 'exit 143' 15", - } + ) lines = append(lines, scheduleRunPreamble(state)...) // The lease is held; the schedule mutex goes back before the first // attempt so a compatible deploy is not blocked through the backoff. @@ -277,9 +298,48 @@ func scheduleRunPreamble(state string) []string { "started_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", "started_epoch=$(date -u '+%s')", "if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=manual; fi", + }, scheduleStateFunction()...) +} + +// scheduleInputsLines consumes the one-shot inputs file on a manual +// activation. Values reach the container as -e arguments, never as shell +// text, and the file is gone before any lock is taken so a skipped manual run +// cannot hand its inputs to the next timer firing. A timer activation never +// opens the file: TRIGGER_UNIT says which one this is. +func scheduleInputsLines(inputsPath string) []string { + return []string{ "operation=''", "inputs_json=''", - }, scheduleStateFunction()...) + "inputs_file=" + q(inputsPath), + "if [ -z \"${TRIGGER_UNIT:-}\" ] && [ -f \"$inputs_file\" ]; then", + " while IFS= read -r line || [ -n \"$line\" ]; do", + " case \"$line\" in", + " ONEBOX_OPERATION=*) operation=${line#ONEBOX_OPERATION=} ;;", + " [A-Z]*=*) set -- \"$@\" -e \"$line\"; key=${line%%=*}; value=${line#*=}; inputs_json=\"${inputs_json:+$inputs_json,}\\\"$key\\\":\\\"$value\\\"\" ;;", + " esac", + " done <\"$inputs_file\"", + " rm -f \"$inputs_file\"", + "fi", + } +} + +func needsTriggerUnit(jobs []app.ScheduledJob) bool { + for _, job := range jobs { + if len(job.Inputs) > 0 { + return true + } + } + return false +} + +// systemdVersion reads the leading number from `systemd 255 (255.4-1ubuntu8)`. +func systemdVersion(firstLine string) (int, bool) { + fields := strings.Fields(firstLine) + if len(fields) < 2 || fields[0] != "systemd" { + return 0, false + } + n, err := strconv.Atoi(fields[1]) + return n, err == nil } // scheduleAttemptLoop runs the container until it exits 0 or the attempts are diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index 0217eb3..f81c838 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -196,7 +196,7 @@ func TestScheduledJobUnitContract(t *testing.T) { "--project-directory", "/var/lib/ob/sample/current", "compose.yaml", - "run --rm --no-deps --name 'sample-nightly-1'", + `run --rm --no-deps "$@" --name 'sample-nightly-1'`, "docker rm -f 'sample-nightly-1'", "nightly", } { @@ -257,7 +257,7 @@ func TestPinnedScheduledJobRunnerLeasesImmutableRelease(t *testing.T) { "--project-directory \"$release_dir\"", "-f \"$release_dir\"/'compose.yaml'", "--env-file \"$release_dir\"/'config/runtime.env'", - "run --rm --no-deps --name 'sample-refresh-1' 'refresh'", + `run --rm --no-deps "$@" --name 'sample-refresh-1' 'refresh'`, "docker rm -f 'sample-refresh-1'", } { if !strings.Contains(runner, want) { @@ -1215,3 +1215,133 @@ func TestScheduledJobNotifierSendsOnlySelectedOutcomesWithTheRunID(t *testing.T) }) } } + +func TestScheduledJobRunnerConsumesManualInputsWithoutShellInterpolation(t *testing.T) { + job := app.ScheduledJob{Name: "sync", Timeout: "45m", DeployLock: "pinned", RetryAttempts: 1, + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog"}, Default: "catalog"}}} + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil) + for _, want := range []string{ + "inputs_file='/var/lib/ob/sample/schedule/sync.inputs'", + `if [ -z "${TRIGGER_UNIT:-}" ] && [ -f "$inputs_file" ]; then`, + `while IFS= read -r line || [ -n "$line" ]; do`, + `ONEBOX_OPERATION=*) operation=${line#ONEBOX_OPERATION=} ;;`, + `[A-Z]*=*) set -- "$@" -e "$line"`, + `rm -f "$inputs_file"`, + `run --rm --no-deps "$@" --name 'sample-sync-1' 'sync'`, + } { + if !strings.Contains(runner, want) { + t.Errorf("runner is missing %q:\n%s", want, runner) + } + } + if strings.Contains(runner, ". \"$inputs_file\"") || strings.Contains(runner, "eval") { + t.Fatalf("runner evaluates the inputs file as shell:\n%s", runner) + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(runner) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("runner is not valid POSIX shell: %v: %s\n%s", err, output, runner) + } + // The consume block precedes the locks so a skipped manual run cannot + // leave its inputs for the next timer firing. + if strings.Index(runner, "inputs_file=") > strings.Index(runner, "flock --exclusive --nonblock --conflict-exit-code 75 9") { + t.Fatalf("inputs are consumed after the lock:\n%s", runner) + } + exclusive := scheduleRunnerScript("sample", app.ScheduledJob{Name: "sync", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil) + if !strings.Contains(exclusive, "inputs_file=") || !strings.Contains(exclusive, `run --rm --no-deps "$@" --name`) { + t.Fatalf("exclusive runner does not consume inputs:\n%s", exclusive) + } +} + +func TestSyncSchedulesRequiresSystemd252ForInputs(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"a"}, Default: "a"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + for version, wantErr := range map[string]bool{"systemd 249 (249.11-0ubuntu3)\n": true, "systemd 255 (255.4-1ubuntu8)\n": false, "": true} { + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "list-unit-files"): + return transport.Result{}, true + case strings.Contains(cmd, "systemd-analyze calendar"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl --version"): + return transport.Result{Stdout: version}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + err := e.SyncSchedules(context.Background()) + if wantErr && (err == nil || !strings.Contains(err.Error(), "systemd 252")) { + t.Fatalf("version %q was accepted for a job with inputs: %v", version, err) + } + if !wantErr && err != nil { + t.Fatalf("version %q was refused: %v", version, err) + } + } +} + +// The inputs block is the one place operator text meets the runner, so it is +// executed rather than only inspected: values with spaces and equals signs +// must arrive as single -e arguments, the metadata line must never become an +// argument, and the file must be gone afterwards. +func TestScheduleInputsLinesParseTheFileIntoArguments(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX shell required") + } + dir := t.TempDir() + inputs := filepath.Join(dir, "sync.inputs") + body := "ONEBOX_OPERATION=20260905-151200-schedule_run-7c1e\nSOURCE=prices and more\nSINCE=2026-09-01=ish\n" + if err := os.WriteFile(inputs, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + script := strings.Join(append(append([]string{"#!/bin/sh", "set -eu"}, scheduleInputsLines(inputs)...), + `printf 'argc=%s\n' "$#"`, + `for arg in "$@"; do printf 'arg=%s\n' "$arg"; done`, + `printf 'operation=%s\n' "$operation"`, + `printf 'json=%s\n' "$inputs_json"`, + ), "\n") + for _, trigger := range []string{"", "ob-sample-sync.timer"} { + command := exec.CommandContext(context.Background(), "sh", "-s") + command.Stdin = strings.NewReader(script) + command.Env = []string{"PATH=" + os.Getenv("PATH")} + if trigger != "" { + command.Env = append(command.Env, "TRIGGER_UNIT="+trigger) + } + out, err := command.CombinedOutput() + if err != nil { + t.Fatalf("trigger %q: %v\n%s", trigger, err, out) + } + got := string(out) + if trigger != "" { + if !strings.Contains(got, "argc=0\n") || !strings.Contains(got, "operation=\n") { + t.Fatalf("a timer activation read the inputs file:\n%s", got) + } + if _, err := os.Stat(inputs); err != nil { + t.Fatalf("a timer activation removed the inputs file: %v", err) + } + continue + } + for _, want := range []string{ + "argc=4\n", "arg=-e\narg=SOURCE=prices and more\n", "arg=-e\narg=SINCE=2026-09-01=ish\n", + "operation=20260905-151200-schedule_run-7c1e\n", + `json="SOURCE":"prices and more","SINCE":"2026-09-01=ish"` + "\n", + } { + if !strings.Contains(got, want) { + t.Fatalf("manual activation output is missing %q:\n%s", want, got) + } + } + if _, err := os.Stat(inputs); err == nil { + t.Fatal("the inputs file survived a manual activation") + } + if err := os.WriteFile(inputs, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } +} From f10a651dad70397c5274324666471499dd6b569b Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:33:03 -0700 Subject: [PATCH 11/25] feat(schedule): operator-initiated runs with validated inputs, journaled as schedule_run Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/audit.go | 4 + internal/engine/schedule_run.go | 173 +++++++++++++++++++++++++++ internal/engine/schedule_run_test.go | 115 ++++++++++++++++++ internal/onebox/binding.go | 2 +- internal/onebox/execute.go | 5 + internal/onebox/execution_types.go | 7 ++ internal/onebox/operation_types.go | 3 +- 7 files changed, 307 insertions(+), 2 deletions(-) create mode 100644 internal/engine/schedule_run.go create mode 100644 internal/engine/schedule_run_test.go diff --git a/internal/engine/audit.go b/internal/engine/audit.go index 6537b2c..2593486 100644 --- a/internal/engine/audit.go +++ b/internal/engine/audit.go @@ -173,6 +173,8 @@ func auditAction(phase string) string { switch phase { case "service-apply": return "service apply" + case "schedule-run": + return "schedule run" case "": return "deploy" default: @@ -190,6 +192,8 @@ func auditOutcome(action string) string { return "applied" case "exec": return "succeeded" + case "schedule run": + return "started" default: return "deployed" } diff --git a/internal/engine/schedule_run.go b/internal/engine/schedule_run.go new file mode 100644 index 0000000..c9da2a6 --- /dev/null +++ b/internal/engine/schedule_run.go @@ -0,0 +1,173 @@ +package engine + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/journal" +) + +// ScheduleRunResult is what an operator-initiated run leaves on the +// workstation side. The outcome lives on the host: it is the run record, and +// it is returned here only when the caller waited for it. +type ScheduleRunResult struct { + Job string `json:"job"` + Unit string `json:"unit"` + Operation string `json:"operation"` + Inputs map[string]string `json:"inputs,omitempty"` + Started bool `json:"started"` + Record *ScheduleRunRecord `json:"record,omitempty"` +} + +// ScheduleRun starts a scheduled job's unit now with declared, validated +// inputs. It is journaled like every operation, but the application lock is +// released before the unit starts: the runner exits 75 when it sees that +// lock, so holding it across the start would skip the very run requested. +// +// Only a job whose data effect is none may run this way. The timer already +// runs any scheduled job unattended, but an operator choosing the moment and +// the inputs is the case the sealed plan of `ob job run` exists for, and a +// migration or destructive job keeps that path. +func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inputs map[string]string, wait bool) (ScheduleRunResult, error) { + result := ScheduleRunResult{Job: name, Operation: operationID, Inputs: inputs} + if strings.TrimSpace(operationID) == "" { + return result, errors.New("schedule run requires an operation id") + } + if err := e.RequireHostOwner(ctx); err != nil { + return result, err + } + if _, err := e.scheduledJob(name); err != nil { + return result, err + } + workload := e.Spec.Workloads[name] + if workload.DataEffect != app.DataEffectNone { + return result, fmt.Errorf("job %s declares data_effect %q; operator-initiated runs of it go through the sealed plan: ob job plan %s, then ob job run", + name, workload.DataEffect, name) + } + if err := app.ValidateJobInputValues(workload, inputs); err != nil { + return result, err + } + unit := e.names().ScheduledJobUnit(name) + result.Unit = unit + + // Starting an active unit is a no-op to systemd and would consume the + // inputs file for a run that never happens; say so instead. + active, err := e.T.Run(ctx, "systemctl is-active "+q(unit+".service")+" 2>/dev/null || true") + if err != nil { + return result, err + } + switch state := strings.TrimSpace(active.Stdout); state { + case "active", "activating", "deactivating": + return result, fmt.Errorf("job %s is running (%s); wait for it, or read ob schedule history %s", name, state, name) + } + + epoch, err := e.AcquireLock(ctx, operationID, e.Opts.ForceLock) + if err != nil { + return result, err + } + locked := true + defer func() { + if locked { + e.ReleaseLock(ctx) + } + }() + if err := e.WriteFence(ctx, operationID, epoch); err != nil { + return result, err + } + + // noclobber: a second manual run before the first is consumed would + // otherwise rewrite the file under it and misattribute the inputs. + path := e.names().ScheduledJobRunInputs(name) + create := "umask 077 && install -d -m 700 " + q(e.names().AppDir()+"/schedule") + " && set -C && cat > " + q(path) + res, err := e.T.RunInput(ctx, create, scheduleInputsFile(operationID, inputs)) + if err != nil { + return result, err + } + if res.ExitCode != 0 { + return result, fmt.Errorf("a manual run of %s is already pending (%s exists); wait for it, or remove the file on the host", name, path) + } + + writer := &journal.Writer{ + T: e.T, Names: e.names(), DeployID: operationID, Epoch: epoch, Operator: journal.DefaultOperator(), + GitSHA: e.Opts.GitSHA, ConfigHash: e.Opts.ConfigHash, Runner: &e.Opts.Runner, + } + detail := "inputs: defaults" + if len(inputs) > 0 { + detail = "inputs: " + scheduleInputsDetail(inputs) + } + record := journal.Record{Phase: "schedule-run", Event: "start", Status: "ok", Target: name, TargetKind: "job", Detail: detail} + if err := writer.Append(ctx, record); err != nil { + return result, fmt.Errorf("journal schedule run start: %w", err) + } + // The finish is written now, under the lock, because the outcome does not + // belong to this operation: it is the run record on the host, joined to + // this journal entry by the operation id the inputs file carries. + record.Event, record.Detail = "finish", "unit started; outcome in ob schedule history "+name + if err := writer.Append(ctx, record); err != nil { + return result, fmt.Errorf("journal schedule run finish: %w", err) + } + e.ReleaseLock(ctx) + locked = false + + start := "systemctl start --no-block " + q(unit+".service") + if wait { + start = "systemctl start " + q(unit+".service") + } + res, err = e.mutate(ctx, start) + if err != nil { + return result, err + } + result.Started = true + if !wait { + if res.ExitCode != 0 { + return result, fmt.Errorf("systemctl start %s: %s", unit, strings.TrimSpace(res.Stderr)) + } + return result, nil + } + records, err := e.ScheduleHistory(ctx, name, 1) + if err != nil { + return result, err + } + if len(records) > 0 { + result.Record = &records[0] + } + // A skipped run exits 75, which SuccessExitStatus makes a clean exit; any + // other non-zero exit is the job failing, and the record says how. + if res.ExitCode != 0 { + return result, fmt.Errorf("job %s did not succeed; see ob schedule logs %s", name, name) + } + return result, nil +} + +// scheduleInputsFile is the one-shot file the runner consumes: the operation +// id on its reserved line, then one declared override per line. Values were +// validated against a charset that has no newline or quote, so the format +// needs no escaping. +func scheduleInputsFile(operationID string, inputs map[string]string) string { + lines := []string{app.ReservedInputPrefix + "OPERATION=" + operationID} + for _, name := range sortedInputNames(inputs) { + lines = append(lines, name+"="+inputs[name]) + } + return strings.Join(lines, "\n") + "\n" +} + +func scheduleInputsDetail(inputs map[string]string) string { + parts := make([]string, 0, len(inputs)) + for _, name := range sortedInputNames(inputs) { + parts = append(parts, name+"="+inputs[name]) + } + return strings.Join(parts, ",") +} + +func sortedInputNames(inputs map[string]string) []string { + names := make([]string, 0, len(inputs)) + for name := range inputs { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/internal/engine/schedule_run_test.go b/internal/engine/schedule_run_test.go new file mode 100644 index 0000000..71855c2 --- /dev/null +++ b/internal/engine/schedule_run_test.go @@ -0,0 +1,115 @@ +package engine + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/transport" +) + +func TestScheduleRunWritesInputsJournalsThenStartsAfterReleasingTheLock(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog", "prices"}, Default: "catalog"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "systemctl start"): + return transport.Result{}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + result, err := e.ScheduleRun(context.Background(), "20260905-151200-schedule_run-7c1e", "sync", map[string]string{"SOURCE": "prices"}, false) + if err != nil { + t.Fatalf("schedule run: %v\n%s", err, strings.Join(f.Commands, "\n")) + } + if !result.Started || result.Unit != "ob-sample-sync" || result.Inputs["SOURCE"] != "prices" || result.Operation != "20260905-151200-schedule_run-7c1e" { + t.Fatalf("result = %#v", result) + } + seq := strings.Join(f.Commands, "\n") + inputs := strings.Index(seq, "sync.inputs") + start := strings.Index(seq, "systemctl start --no-block 'ob-sample-sync.service'") + release := strings.LastIndex(seq, "rm -f '/var/lib/ob/sample/lock'") + if inputs < 0 || start < 0 || release < 0 || !(inputs < release && release < start) { + t.Fatalf("expected inputs write, lock release, then start:\n%s", seq) + } + if !strings.Contains(seq, "set -C") { + t.Fatalf("inputs file was not created with noclobber:\n%s", seq) + } + if written := strings.Join(f.Inputs, "\n"); !strings.Contains(written, "ONEBOX_OPERATION=20260905-151200-schedule_run-7c1e\nSOURCE=prices\n") { + t.Fatalf("inputs file content is wrong:\n%s", written) + } + for _, want := range []string{ + `"phase":"schedule-run","event":"start"`, + `"phase":"schedule-run","event":"finish","status":"ok"`, + `"target":"sync"`, + `inputs: SOURCE=prices`, + } { + if !strings.Contains(seq, want) { + t.Fatalf("journal is missing %q:\n%s", want, seq) + } + } + if journal := strings.Index(seq, `"phase":"schedule-run","event":"finish"`); journal > release { + t.Fatalf("journal finish was written after the lock was released:\n%s", seq) + } +} + +func TestScheduleRunRefusals(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog"}, Default: "catalog"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + cfg.Workloads["prune"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "destructive", + Schedule: &app.JobSchedule{Cron: "0 3 * * *", Timezone: "UTC", Timeout: "1h"}, + } + active := false + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "command -v flock") { + return transport.Result{Stdout: "ok\n"}, true + } + if strings.Contains(cmd, "systemctl is-active") { + if active { + return transport.Result{Stdout: "activating\n"}, true + } + return transport.Result{Stdout: "inactive\n"}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + ctx := context.Background() + if _, err := e.ScheduleRun(ctx, "op", "prune", nil, false); err == nil || !strings.Contains(err.Error(), "ob job run") { + t.Fatalf("destructive job accepted: %v", err) + } + if _, err := e.ScheduleRun(ctx, "op", "sync", map[string]string{"SOURCE": "reviews"}, false); err == nil { + t.Fatal("undeclared value accepted") + } + if _, err := e.ScheduleRun(ctx, "op", "web", nil, false); err == nil { + t.Fatal("non-scheduled workload accepted") + } + for _, command := range f.Commands { + if strings.Contains(command, "systemctl start") || strings.Contains(command, ".inputs") { + t.Fatalf("a refused run reached the host: %s", command) + } + } + active = true + if _, err := e.ScheduleRun(ctx, "op", "sync", nil, false); err == nil || !strings.Contains(err.Error(), "running") { + t.Fatalf("active unit not refused: %v", err) + } +} diff --git a/internal/onebox/binding.go b/internal/onebox/binding.go index 66a8276..d40c1f6 100644 --- a/internal/onebox/binding.go +++ b/internal/onebox/binding.go @@ -31,7 +31,7 @@ func (s *Service) ResolveExecutionBinding(ctx context.Context, kind OperationKin func operationUsesInspectionRuntime(kind OperationKind) bool { switch kind { case KindResume, KindAbort, KindRollback, KindBootstrap, KindServiceApply, - KindProxyApply, KindScheduleApply, KindSecretsPush, KindDestroy, + KindProxyApply, KindScheduleApply, KindScheduleRun, KindSecretsPush, KindDestroy, // Backup operates on a service's data, never on the application's // release images, so a placeholder image must not stop a backup. KindBackupEnable, KindBackupDisable, KindBackupCreate, KindBackupPrune, KindAssuranceCheck, diff --git a/internal/onebox/execute.go b/internal/onebox/execute.go index 9928da2..d40864e 100644 --- a/internal/onebox/execute.go +++ b/internal/onebox/execute.go @@ -208,6 +208,11 @@ func (s *Service) Execute(ctx context.Context, request ExecuteRequest) (Operatio case KindScheduleApply: result.EvidenceID = operationID err = e.ScheduleApply(ctx, operationID) + case KindScheduleRun: + result.EvidenceID = operationID + var run engine.ScheduleRunResult + run, err = e.ScheduleRun(ctx, operationID, request.Job, request.Inputs, request.Wait) + result.ScheduleRun = &run case KindSecretsPush: entries := encryptedEntries(lp.resolved) externalProjections := externalConnectionProjections(lp.resolved) diff --git a/internal/onebox/execution_types.go b/internal/onebox/execution_types.go index 7db7148..414f8b7 100644 --- a/internal/onebox/execution_types.go +++ b/internal/onebox/execution_types.go @@ -294,6 +294,12 @@ type ExecuteRequest struct { MigrationBackupOverride *MigrationBackupOverride BreakLock bool AllowDestructiveMounts bool + // Job, Inputs and Wait are the schedule_run arguments: a declared + // scheduled job, validated input overrides, and whether to block until + // the unit exits and return its run record. + Job string + Inputs map[string]string + Wait bool // Service is the backup operations' one argument. It is an input to a // mutation rather than a plan, because a backup stages nothing into a // release and has nothing to roll back. @@ -370,6 +376,7 @@ type OperationResult struct { MigrationBackupOverrideDigest string `json:"migration_backup_override_digest,omitempty"` Runner buildinfo.Runner `json:"runner"` JobResult *journal.JobResultEvidence `json:"job_result,omitempty"` + ScheduleRun *engine.ScheduleRunResult `json:"schedule_run,omitempty"` } func validateRunnerProvenance(runner buildinfo.Runner, planSchema string) error { diff --git a/internal/onebox/operation_types.go b/internal/onebox/operation_types.go index 101ca69..26e6e6f 100644 --- a/internal/onebox/operation_types.go +++ b/internal/onebox/operation_types.go @@ -41,6 +41,7 @@ const ( KindSecretsPush OperationKind = "secrets_push" KindDestroy OperationKind = "destroy" KindJobRun OperationKind = "job_run" + KindScheduleRun OperationKind = "schedule_run" KindServiceImagePatch OperationKind = "service_image_patch" KindBackupEnable OperationKind = "backup_enable" @@ -477,7 +478,7 @@ func requireJSONEOF(decoder *json.Decoder) error { func validOperationKind(kind OperationKind) bool { switch kind { case KindDeploy, KindResume, KindAbort, KindRollback, KindBootstrap, KindJobRun, - KindServiceApply, KindProxyApply, KindScheduleApply, KindSecretsPush, KindDestroy, + KindServiceApply, KindProxyApply, KindScheduleApply, KindScheduleRun, KindSecretsPush, KindDestroy, KindServiceImagePatch, KindBackupEnable, KindBackupDisable, KindBackupCreate, KindBackupPrune, KindReplayArchive, KindRestoreTest, KindRestorePrepare, KindRestoreCutover, KindRestoreAbort, From 1e818f8862fd872250aeec97e10652f5148ab3de Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:34:24 -0700 Subject: [PATCH 12/25] feat(cli): ob schedule run starts a scheduled job now with validated inputs Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- cmd/ob/output.go | 1 + cmd/ob/output_test.go | 1 + cmd/ob/schedule.go | 42 ++++++++++++++++++++ cmd/ob/schedule_test.go | 18 +++++++++ internal/engine/schedule_run.go | 7 ++++ internal/onebox/operation_errors.go | 4 ++ site/src/content/docs/reference/cli.mdx | 26 ++++++++++++ site/src/content/docs/reference/errors.mdx | 1 + site/src/content/docs/reference/policies.mdx | 2 +- 9 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 cmd/ob/schedule_test.go diff --git a/cmd/ob/output.go b/cmd/ob/output.go index e1b9c4f..8681077 100644 --- a/cmd/ob/output.go +++ b/cmd/ob/output.go @@ -108,6 +108,7 @@ var cliOutputMatrix = map[string]cliOutputClass{ "ob schedule history": {Class: cliClassFiniteEnvelope, JSON: true}, "ob schedule list": {Class: cliClassFiniteEnvelope, JSON: true}, "ob schedule logs": {Class: cliClassOperatorPassthrough, JSON: true, NDJSON: true}, + "ob schedule run": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, "ob schema": {Class: cliClassFiniteEnvelope, JSON: true}, "ob secrets edit": {Class: cliClassTrustedEditor, JSON: true}, "ob secrets list": {Class: cliClassFiniteEnvelope, JSON: true}, diff --git a/cmd/ob/output_test.go b/cmd/ob/output_test.go index 88b2b23..41e5a11 100644 --- a/cmd/ob/output_test.go +++ b/cmd/ob/output_test.go @@ -522,6 +522,7 @@ func TestLeafOutputMatrixIsClosedAndHasNoAliases(t *testing.T) { "ob schedule history": {Class: "finite_envelope", JSON: true}, "ob schedule list": {Class: "finite_envelope", JSON: true}, "ob schedule logs": {Class: "operator_passthrough", JSON: true, NDJSON: true}, + "ob schedule run": {Class: "finite_stream", JSON: true, NDJSON: true}, "ob schema": {Class: "finite_envelope", JSON: true}, "ob secrets edit": {Class: "trusted_editor", JSON: true}, "ob secrets list": {Class: "finite_envelope", JSON: true}, diff --git a/cmd/ob/schedule.go b/cmd/ob/schedule.go index ef7005f..7b62883 100644 --- a/cmd/ob/schedule.go +++ b/cmd/ob/schedule.go @@ -170,9 +170,51 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { logsCmd.Flags().IntVarP(&logsTail, "tail", "n", 200, "lines to show when no run is recorded") scheduleCmd.AddCommand(logsCmd) + var runInputs []string + var runWait, runBreakLock bool + runCmd := &cobra.Command{ + Use: "run ", + Short: "start a scheduled job now with declared inputs", + Long: "Start one scheduled job's unit now, with values for its declared inputs. Values are validated on the workstation against the declaration; an undeclared name or a value outside its enum or pattern is refused before anything reaches the host.\n\n" + + "Only a job with data_effect none may run this way; a migration or destructive job keeps the sealed plan of ob job run. The request is journaled as schedule_run with the operator and inputs, and the host record carries the operation id, so ob audit and ob schedule history join on it.\n\n" + + "The outcome is the run record: ob schedule history , or --wait to block until the unit exits and print it.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + inputs, err := parseScheduleInputs(runInputs) + if err != nil { + return writeEarlyOperationFailure(cmd, g, codedError("schedule_input_invalid", "%v", err)) + } + return runMutation(cmd, g, onebox.ExecuteRequest{ + Kind: onebox.KindScheduleRun, Job: args[0], Inputs: inputs, Wait: runWait, BreakLock: runBreakLock, + }, "schedule run") + }, + } + runCmd.Flags().StringArrayVar(&runInputs, "input", nil, "input override as NAME=VALUE; repeatable") + runCmd.Flags().BoolVar(&runWait, "wait", false, "block until the unit exits and report the run record") + runCmd.Flags().BoolVar(&runBreakLock, "break-lock", false, "break a stale operation lock after inspecting its holder") + scheduleCmd.AddCommand(runCmd) + root.AddCommand(scheduleCmd) } +// parseScheduleInputs turns repeated --input NAME=VALUE flags into overrides. +// Only the shape is checked here; names and values are validated against the +// job's declaration by the engine before anything reaches the host. +func parseScheduleInputs(raw []string) (map[string]string, error) { + out := map[string]string{} + for _, item := range raw { + name, value, ok := strings.Cut(item, "=") + if !ok || name == "" { + return nil, fmt.Errorf("--input %q must be NAME=VALUE", item) + } + if _, dup := out[name]; dup { + return nil, fmt.Errorf("--input %s given twice", name) + } + out[name] = value + } + return out, nil +} + func orDash(s string) string { if strings.TrimSpace(s) == "" { return "-" diff --git a/cmd/ob/schedule_test.go b/cmd/ob/schedule_test.go new file mode 100644 index 0000000..4076d5f --- /dev/null +++ b/cmd/ob/schedule_test.go @@ -0,0 +1,18 @@ +package main + +import "testing" + +func TestParseScheduleInputsFlags(t *testing.T) { + got, err := parseScheduleInputs([]string{"SOURCE=prices", "SINCE=2026-09-01=ish"}) + if err != nil || got["SOURCE"] != "prices" || got["SINCE"] != "2026-09-01=ish" { + t.Fatalf("got %#v, %v", got, err) + } + if got, err := parseScheduleInputs(nil); err != nil || len(got) != 0 { + t.Fatalf("no flags should mean no overrides: %#v, %v", got, err) + } + for _, bad := range [][]string{{"SOURCE"}, {"=x"}, {"SOURCE=a", "SOURCE=b"}} { + if _, err := parseScheduleInputs(bad); err == nil { + t.Errorf("%v was accepted", bad) + } + } +} diff --git a/internal/engine/schedule_run.go b/internal/engine/schedule_run.go index c9da2a6..122573e 100644 --- a/internal/engine/schedule_run.go +++ b/internal/engine/schedule_run.go @@ -126,6 +126,7 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu if res.ExitCode != 0 { return result, fmt.Errorf("systemctl start %s: %s", unit, strings.TrimSpace(res.Stderr)) } + e.logf("schedule: %s started as %s; ob schedule history %s shows the outcome", name, operationID, name) return result, nil } records, err := e.ScheduleHistory(ctx, name, 1) @@ -134,6 +135,12 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu } if len(records) > 0 { result.Record = &records[0] + exit := "-" + if records[0].ExitStatus != nil { + exit = fmt.Sprint(*records[0].ExitStatus) + } + e.logf("schedule: %s run %s %s after %d attempt(s) in %ds (exit %s)", + name, records[0].Run, records[0].Outcome, records[0].Attempts, records[0].DurationSeconds, exit) } // A skipped run exits 75, which SuccessExitStatus makes a clean exit; any // other non-zero exit is the job failing, and the record says how. diff --git a/internal/onebox/operation_errors.go b/internal/onebox/operation_errors.go index 1548065..deda70a 100644 --- a/internal/onebox/operation_errors.go +++ b/internal/onebox/operation_errors.go @@ -125,6 +125,10 @@ var operationFailureDefinitions = map[string]OperationFailure{ Message: "the scheduled job's run records could not be read from the host journal", Command: "ob status --output json", }, + "schedule_input_invalid": { + Message: "an --input flag is not NAME=VALUE, or names the same input twice", + Command: "ob canonical --output json", + }, "schedule_list_failed": { Message: "the scheduled jobs' timer state could not be read", Command: "ob status --output json", diff --git a/site/src/content/docs/reference/cli.mdx b/site/src/content/docs/reference/cli.mdx index c01cc07..d1d7684 100644 --- a/site/src/content/docs/reference/cli.mdx +++ b/site/src/content/docs/reference/cli.mdx @@ -941,6 +941,7 @@ Available Commands: history run records of one scheduled job, newest first list declared scheduled jobs with timer state and next elapse logs journal of one scheduled run + run start a scheduled job now with declared inputs Flags: -h, --help help for schedule @@ -1037,6 +1038,31 @@ Global Flags: -v, --verbose print every remote command ``` +### ob schedule run + +``` +Start one scheduled job's unit now, with values for its declared inputs. Values are validated on the workstation against the declaration; an undeclared name or a value outside its enum or pattern is refused before anything reaches the host. + +Only a job with data_effect none may run this way; a migration or destructive job keeps the sealed plan of ob job run. The request is journaled as schedule_run with the operator and inputs, and the host record carries the operation id, so ob audit and ob schedule history join on it. + +The outcome is the run record: ob schedule history , or --wait to block until the unit exits and print it. + +Usage: + ob schedule run [flags] + +Flags: + --break-lock break a stale operation lock after inspecting its holder + -h, --help help for run + --input stringArray input override as NAME=VALUE; repeatable + --wait block until the unit exits and report the run record + +Global Flags: + -c, --config string path to the project YAML file (default "ob.yml") + -e, --env string environment name (default "production") + --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") + -v, --verbose print every remote command +``` + ## ob schema ``` diff --git a/site/src/content/docs/reference/errors.mdx b/site/src/content/docs/reference/errors.mdx index 21b585c..acd1493 100644 --- a/site/src/content/docs/reference/errors.mdx +++ b/site/src/content/docs/reference/errors.mdx @@ -153,6 +153,7 @@ step to complete rather than a line to run verbatim. | `recovery_incomplete` | recovery did not reach its verified terminal state | resolving | `ob resume --output ndjson` | | `rollback_target_missing` | no previously serving release is recorded as a rollback target | next | `ob plan --output json` | | `schedule_history_failed` | the scheduled job's run records could not be read from the host journal | diagnostic | `ob status --output json` | +| `schedule_input_invalid` | an --input flag is not NAME=VALUE, or names the same input twice | diagnostic | `ob canonical --output json` | | `schedule_list_failed` | the scheduled jobs' timer state could not be read | diagnostic | `ob status --output json` | | `schedule_logs_failed` | the scheduled run's journal could not be read | next | `ob schedule history --output json` | | `secret_cleanup_pending` | the rotation is applied and verified, but removing the retired generation did not finish | resolving | `ob secrets push --output ndjson` | diff --git a/site/src/content/docs/reference/policies.mdx b/site/src/content/docs/reference/policies.mdx index 03ba005..a1fd650 100644 --- a/site/src/content/docs/reference/policies.mdx +++ b/site/src/content/docs/reference/policies.mdx @@ -97,7 +97,7 @@ redacted. | Class | JSON | NDJSON | Commands | | --- | --- | --- | --- | | Finite envelope | yes | no | `ob approve` · `ob audit` · `ob backup status` · `ob canonical` · `ob doctor` · `ob eject` · `ob init` · `ob job plan` · `ob plan` · `ob preflight` · `ob preview` · `ob schedule history` · `ob schedule list` · `ob schema` · `ob secrets list` · `ob status` · `ob validate` · `ob version` | -| Finite operation stream | yes | yes | `ob abort` · `ob backup create` · `ob backup enable` · `ob backup disable` · `ob backup drill` · `ob backup prune` · `ob backup restore` · `ob backup verify` · `ob bootstrap` · `ob deploy` · `ob destroy` · `ob job run` · `ob proxy apply` · `ob resume` · `ob rollback` · `ob schedule apply` · `ob secrets push` · `ob service apply` | +| Finite operation stream | yes | yes | `ob abort` · `ob backup create` · `ob backup enable` · `ob backup disable` · `ob backup drill` · `ob backup prune` · `ob backup restore` · `ob backup verify` · `ob bootstrap` · `ob deploy` · `ob destroy` · `ob job run` · `ob proxy apply` · `ob resume` · `ob rollback` · `ob schedule apply` · `ob schedule run` · `ob secrets push` · `ob service apply` | | Operator passthrough | finite only | yes | `ob logs` · `ob schedule logs` | | Operator passthrough | no | yes | `ob exec` | | Trusted editor | yes, after exit | no | `ob secrets edit` | From 4525c5b08d6caef0e20babfc5cca701a9cad2747 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:37:08 -0700 Subject: [PATCH 13/25] docs(site): retry, run records, notify and manual runs for scheduled jobs Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- .../content/docs/guides/schedule-a-job.mdx | 161 ++++++++++++++++-- site/src/content/docs/status/capabilities.mdx | 5 + 2 files changed, 150 insertions(+), 16 deletions(-) diff --git a/site/src/content/docs/guides/schedule-a-job.mdx b/site/src/content/docs/guides/schedule-a-job.mdx index 527ee7f..922ebde 100644 --- a/site/src/content/docs/guides/schedule-a-job.mdx +++ b/site/src/content/docs/guides/schedule-a-job.mdx @@ -1,11 +1,13 @@ --- title: Schedule a job -description: Bounded host timers, safe deployment coordination, failure status, and the cron forms Onebox refuses. -summary: How to declare a scheduled job, bound its run time, choose deployment and catch-up behavior, inspect failures, and understand which cron expressions are refused at load. +description: Bounded host timers, retry inside one firing, a record for every run, deployment coordination, manual runs with declared inputs, and the cron forms Onebox refuses. +summary: How to declare a scheduled job, bound its run time and retries, choose deployment and catch-up behavior, read run records back, run one now with inputs, and understand which cron expressions are refused at load. sidebar: order: 4 read_when: - "Adding a nightly or recurring task" + - "Working out why a scheduled run failed, or whether it ran at all" + - "Running a scheduled job now with different parameters" - "Understanding why a cron expression was refused" - "Working out how a scheduled job resolves secrets with no Onebox process running" --- @@ -42,8 +44,33 @@ result. Set a longer duration for jobs that legitimately need it. `catch_up` defaults to `true`: if the host was off at the scheduled time, the timer runs once after it returns. Set it to `false` for time-sensitive work that -should be skipped rather than run late. A failed run is not retried implicitly; -the next attempt is the next cron elapse. +should be skipped rather than run late. + +## Retry inside one firing + +By default a failed run is not retried; the next attempt is the next cron +elapse, which is the right answer for anything that fires every few minutes. +For a job whose failures are usually transient, a rate limit or an upstream +that was busy for a moment, declare a bounded retry: + +```yaml +schedule: + cron: "0 * * * *" + timeout: 45m + retry: {attempts: 3, backoff: 30s, max_backoff: 10m} +``` + +`attempts` counts every attempt including the first, so the default of `1` is +today's single run. After a non-zero exit the runner sleeps `backoff`, doubles +it after each further failure, and never sleeps longer than `max_backoff`. Every +attempt runs inside the same timer firing, under the same locks and the same +`timeout`; when the timeout expires the run ends, with no further attempt. + +Validation refuses a retry whose worst-case backoff is not smaller than the +timeout, because the last attempt could never start and the record would say +it did. With `deploy_lock: exclusive` the deploy lock is held through the +sleeps, so a deploy waits for the run to finish; with `deploy_lock: pinned` the +release lease is held instead and every attempt runs the same release. ## Deployment coordination is exclusive by default @@ -102,23 +129,68 @@ under the same application lock, schedule mutex, fence, and journal boundary as a deploy, but it does not stage or activate a release. Run it after upgrading Onebox when an application may not be deployed again soon. -## Failures remain visible +## Every run leaves a record -systemd retains the last oneshot result. `ob status` reads it alongside the -timer state; a non-success result or inactive timer is reported as divergence: +When a run ends, for any reason, the unit's `ExecStopPost` writes one record +to the host journal under the job's unit, with the syslog identifier `ob-run`: + +```json +{"run":"a3f9…","job":"nightly-dump","trigger":"timer","operation":"","release":"20260905-140000-ab12cd","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:04:37Z","duration_s":276,"attempts":2,"exit_status":0,"outcome":"success","inputs":{}} +``` + +`run` is systemd's invocation id, so the record and the run's own log share a +key. `outcome` is one of `success`, `failure`, `timeout`, or `skipped`. A skip +is a firing that met a running instance of the same job or an application +operation holding the deploy lock; it exits `75`, which the unit treats as a +clean exit, and it is recorded because a job that is silently never running +looks exactly like one that works. + +Read the records back from the workstation: + +```sh +ob schedule list # every job, its timer state and next elapse +ob schedule history nightly-dump # records, newest first; -n 50 for more +ob schedule logs nightly-dump # the journal of the newest run +ob schedule logs nightly-dump --run a3f9… +``` + +`ob status` reads the same records. Each scheduled job's line carries the next +elapse, the last outcome with its duration and attempt count, and, while a run +is in progress, the attempt it is on. A last outcome of `failure` or `timeout` +is reported as divergence: ```text -schedule nightly-dump last run failed: timeout (exit 15) ⚠ +schedule nightly-dump last run failed: timeout (exit 143) ⚠ ``` -Use `journalctl -u ob--.service` on the target for trusted diagnostic -output. Every configured notification selecting `failure` also receives a -bounded, fail-open POST when a host-fired run fails or times out. Onebox writes -the webhook handler root-only beside the unit, so credentials in webhook -paths do not appear in `ExecStart` or `ExecStopPost`. A delivery failure is -recorded in the unit's journal and never replaces the job's original result. -The webhook must be reachable from the target host; timer delivery uses its -network path, not the operator workstation's. +Retention is the journal's. A host whose journal lives in memory keeps records +only since its last boot; `ob status` says so on the job's line, and the fix is +a persistent journal (`/var/log/journal`), which the supported images have by +default. + +### Notifications per outcome + +`schedule.notify` selects which outcomes send the configured notifications. The +default is `[failure, timeout]`, which is what the failure notifier always did. +Add `success` for a job whose completion someone waits for, and `skipped` for a +job whose firings collide often enough that silence would hide it: + +```yaml +schedule: + cron: "0 2 * * *" + notify: [failure, timeout, skipped] +``` + +Each selected outcome sends a bounded, fail-open POST to every webhook whose +own `on` list accepts that class of outcome. The payload carries the run id as +its `deploy_id`, so `ob schedule logs --run ` finds the run; it +carries nothing else about the run, because notifications cross the host trust +boundary and diagnostics stay on the host. Onebox writes the webhook handler +root-only beside the unit, so credentials in webhook paths do not appear in +`ExecStart` or `ExecStopPost`. A delivery failure is recorded in the unit's +journal and never replaces the job's own result. The webhook must be reachable +from the target host; timer delivery uses its network path, not the operator +workstation's. ## Cron is translated exactly, or refused @@ -140,6 +212,63 @@ load rather than running on days nobody chose. `none`; a nightly prune is `destructive`. The rollback and abort gates read it, and a job that lies about it defeats them. +## Run one now with inputs + +A scheduled job may declare inputs: named parameters that reach the container +as environment variables. A timer firing uses the defaults; an operator may run +the job now and override them. + +```yaml +workloads: + source-sync: + role: job + image: ghcr.io/acme/ingest:1.8.2 + command: ["./ingest", "sync"] + data_effect: none + inputs: + SOURCE: + enum: [catalog, prices, reviews] + default: catalog + description: Which upstream to sync. + SINCE: + pattern: '^([0-9]{4}-[0-9]{2}-[0-9]{2})?$' + default: "" + description: Only records changed since this date. Empty means the stored cursor. + schedule: + cron: "0 * * * *" + deploy_lock: pinned +``` + +```sh +ob schedule run source-sync --input SOURCE=prices --input SINCE=2026-09-01 --wait +``` + +Names are upper-case identifiers outside Onebox's `ONEBOX_` namespace and may +not repeat an `env` key. Each input declares exactly one of `enum` or +`pattern`, and a `default` that satisfies it; a pattern matches the whole +value. Whatever the pattern allows, a value may not contain a double quote, a +backslash, or a control character, and is at most 256 bytes. Those rules are +what let the runner hand values to the container without escaping anything. + +`ob schedule run` validates every value on the workstation, refuses if the +unit is already running or a manual run is still pending, journals the request +as `schedule_run` with the operator and the inputs, and starts the unit. It +holds the application lock only while writing and journaling, because the +runner skips a run that meets that lock. The host record of the run carries the +operation id, so `ob audit` and `ob schedule history` join on it. Without +`--wait` the command returns as soon as the unit is started; with it, the +command blocks until the unit exits and reports the record. + +Only a job with `data_effect: none` accepts inputs or manual runs this way. A +`migration` or `destructive` job is still run by its timer as declared, but an +operator-initiated run of it keeps the sealed plan, approval and backup-report +gates of `ob job run` below. + +The runner tells a manual activation from a timer firing by the `TRIGGER_UNIT` +variable systemd sets on timer activations, which needs systemd 252 or newer +on the host. Ubuntu 24.04 and Debian 12 qualify; a job that declares inputs is +refused at deploy time on an older host. + ## Running one by hand ```sh diff --git a/site/src/content/docs/status/capabilities.mdx b/site/src/content/docs/status/capabilities.mdx index 7c99867..556a78e 100644 --- a/site/src/content/docs/status/capabilities.mdx +++ b/site/src/content/docs/status/capabilities.mdx @@ -27,6 +27,11 @@ This page is the reconciliation. Three states: additive. - Compose generation and validation, SSH transport with known-host checking, state-bound plans, image pinning, rendered diffs. +- Scheduled jobs as host timers with exact cron translation, bounded retry + inside one firing, one run record per activation in the host journal + (`ob schedule history`, `ob schedule logs`, `ob status`), per-outcome + notifications, and declared inputs for operator-initiated runs + (`ob schedule run`). - Per-workload reconciliation that retains proven-unchanged healthy workloads, health-gated rolling or recreate replacement for changed or uncertain ones, traffic drain, verification, versioned releases, retention, rollback. From 3c2a092e316210be2ba3e7e7fb4549c0de78c59e Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 15:39:38 -0700 Subject: [PATCH 14/25] fix(schedule): guard the cleanup trap before the state path is set Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index d39e081..88d3e01 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -212,7 +212,7 @@ func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Na "release_dir=$(readlink -f "+q(names.CurrentLink())+" 2>/dev/null || true)", "release=${release_dir##*/}", scheduleContainerCleanup(container), - "cleanup() { "+scheduleContainerCleanup(container)+"; rm -f \"$tmp\"; }", + "cleanup() { "+scheduleContainerCleanup(container)+"; [ -z \"${tmp:-}\" ] || rm -f \"$tmp\"; }", "trap cleanup 0", "trap 'exit 129' 1", "trap 'exit 130' 2", @@ -254,7 +254,7 @@ func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names "chmod 600 \"$release_dir/.ob-schedule.lease\"", "/usr/bin/flock --shared 7", scheduleContainerCleanup(container), - "cleanup() { "+scheduleContainerCleanup(container)+"; rm -f \"$tmp\"; }", + "cleanup() { "+scheduleContainerCleanup(container)+"; [ -z \"${tmp:-}\" ] || rm -f \"$tmp\"; }", "trap cleanup 0", "trap 'exit 129' 1", "trap 'exit 130' 2", From 573eb52993f6351322cb125dc220eac7df999118 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 16:20:14 -0700 Subject: [PATCH 15/25] fix(schedule): write the run record with explicit journal fields and query by them A process that writes one line and exits is often gone before journald reads /proc for it, so the systemd-cat record arrived with no _SYSTEMD_UNIT and journalctl -u never found it. logger --journald carries ONEBOX_UNIT and ONEBOX_JOB in the entry itself, and the history query matches on those. Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- cmd/ob/schedule.go | 2 +- internal/engine/schedule.go | 23 ++++++++++++------- internal/engine/schedule_history.go | 4 +++- internal/engine/schedule_history_test.go | 4 ++-- internal/engine/schedule_test.go | 22 ++++++++++++------ .../content/docs/guides/schedule-a-job.mdx | 4 +++- site/src/content/docs/reference/cli.mdx | 2 +- 7 files changed, 40 insertions(+), 21 deletions(-) diff --git a/cmd/ob/schedule.go b/cmd/ob/schedule.go index 7b62883..8350075 100644 --- a/cmd/ob/schedule.go +++ b/cmd/ob/schedule.go @@ -78,7 +78,7 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { historyCmd := &cobra.Command{ Use: "history ", Short: "run records of one scheduled job, newest first", - Long: "Read the run records the host wrote for one scheduled job. Each record is one activation: run id, trigger, release, start and end, attempts, exit status, outcome and, for a manual run, its inputs.\n\nRecords live in the host journal under the job's unit with syslog identifier ob-run; retention is the journal's. Reads only.", + Long: "Read the run records the host wrote for one scheduled job. Each record is one activation: run id, trigger, release, start and end, attempts, exit status, outcome and, for a manual run, its inputs.\n\nRecords live in the host journal with syslog identifier ob-run and the job's unit in their ONEBOX_UNIT field; retention is the journal's. Reads only.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { cfg, p, err := loadAllLenient(cmd.Context(), g) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 88d3e01..7588604 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -408,9 +408,15 @@ func scheduleServiceUnit(application string, job app.ScheduledJob, runnerPath, n const scheduleNotificationTimestamp = "__ONEBOX_SCHEDULE_TIMESTAMP__" // scheduleRunIdentifier is the syslog identifier of the one line the notifier -// writes per run. `journalctl -u -t ob-run` is the run history: the -// journal is the store, so there is no file to trim and nothing that can -// disagree with the unit's own log. +// writes per run. The journal is the store, so there is no file to trim and +// nothing that can disagree with the unit's own log. +// +// The line carries its own ONEBOX_UNIT and ONEBOX_JOB fields and the history +// query matches on them, not on journald's cgroup attribution. A process that +// writes one line and exits is often gone before journald reads /proc for it, +// and such an entry has no _SYSTEMD_UNIT at all; `journalctl -u` would never +// find it. Explicit fields survive that race, and `logger --journald` is +// util-linux, which flock already requires. const scheduleRunIdentifier = "ob-run" // scheduleRunRecordLines finalises the run the runner started. This lives in @@ -419,7 +425,7 @@ const scheduleRunIdentifier = "ob-run" // interpolated into the JSON is either numeric, a timestamp the runner // formatted, a release id, or an input value the loader restricted to a // charset that needs no escaping. -func scheduleRunRecordLines(job, state string) []string { +func scheduleRunRecordLines(application, unit, job, state string) []string { return []string{ "state=" + q(state), "release=''; started_at=''; started_epoch=''; trigger=''; operation=''; attempt=0; inputs=''", @@ -452,9 +458,10 @@ func scheduleRunRecordLines(job, state string) []string { "duration=0", "case \"$started_epoch\" in ''|*[!0-9]*) ;; *) duration=$((now - started_epoch)) ;; esac", "[ -z \"$started_at\" ] && started_at=$finished_at", - "printf '{\"run\":\"%s\",\"job\":\"%s\",\"trigger\":\"%s\",\"operation\":\"%s\",\"release\":\"%s\",\"started_at\":\"%s\",\"finished_at\":\"%s\",\"duration_s\":%s,\"attempts\":%s,\"exit_status\":%s,\"outcome\":\"%s\",\"inputs\":{%s}}\\n' " + - "\"${INVOCATION_ID:-}\" " + q(job) + " \"$trigger\" \"$operation\" \"$release\" \"$started_at\" \"$finished_at\" \"$duration\" \"$attempt\" \"$status\" \"$outcome\" \"$inputs\" " + - "| systemd-cat -t " + scheduleRunIdentifier + " || true", + "record=$(printf '{\"run\":\"%s\",\"job\":\"%s\",\"trigger\":\"%s\",\"operation\":\"%s\",\"release\":\"%s\",\"started_at\":\"%s\",\"finished_at\":\"%s\",\"duration_s\":%s,\"attempts\":%s,\"exit_status\":%s,\"outcome\":\"%s\",\"inputs\":{%s}}' " + + "\"${INVOCATION_ID:-}\" " + q(job) + " \"$trigger\" \"$operation\" \"$release\" \"$started_at\" \"$finished_at\" \"$duration\" \"$attempt\" \"$status\" \"$outcome\" \"$inputs\")", + "printf 'MESSAGE=%s\\nPRIORITY=6\\nSYSLOG_IDENTIFIER=" + scheduleRunIdentifier + "\\nONEBOX_APP=%s\\nONEBOX_UNIT=%s\\nONEBOX_JOB=%s\\n' " + + "\"$record\" " + q(application) + " " + q(unit) + " " + q(job) + " | logger --journald || true", } } @@ -487,7 +494,7 @@ func (e *Engine) scheduleNotifier(job app.ScheduledJob) (string, error) { " " + scheduleContainerCleanup(e.names().Container(job.Name, 1)), "fi", } - lines = append(lines, scheduleRunRecordLines(job.Name, e.names().ScheduledJobRunState(job.Name))...) + lines = append(lines, scheduleRunRecordLines(e.Spec.Name, e.names().ScheduledJobUnit(job.Name), job.Name, e.names().ScheduledJobRunState(job.Name))...) lines = append(lines, `case " `+strings.Join(job.Notify, " ")+` " in *" $outcome "*) ;; *) exit 0 ;; esac`) wantsSuccess, wantsFailure := false, false for _, outcome := range job.Notify { diff --git a/internal/engine/schedule_history.go b/internal/engine/schedule_history.go index 2a82c89..7483b5a 100644 --- a/internal/engine/schedule_history.go +++ b/internal/engine/schedule_history.go @@ -47,11 +47,13 @@ type ScheduleListing struct { // match, so it is checked against the only shape systemd produces. var scheduleRunID = regexp.MustCompile(`^[0-9a-f]{32}$`) +// scheduleHistoryCommand matches the record's own fields rather than the +// unit journald attributed it to; see scheduleRunIdentifier for why. func scheduleHistoryCommand(unit string, n int) string { if n <= 0 { n = 20 } - return "journalctl -u " + q(unit+".service") + " -t " + scheduleRunIdentifier + + return "journalctl SYSLOG_IDENTIFIER=" + scheduleRunIdentifier + " ONEBOX_UNIT=" + q(unit) + " -o cat -r -n " + strconv.Itoa(n) + " --no-pager 2>/dev/null || true" } diff --git a/internal/engine/schedule_history_test.go b/internal/engine/schedule_history_test.go index 5777c23..e038da2 100644 --- a/internal/engine/schedule_history_test.go +++ b/internal/engine/schedule_history_test.go @@ -56,7 +56,7 @@ func TestScheduleHistoryReadsTheUnitJournalNewestFirst(t *testing.T) { t.Fatalf("records = %#v", records) } seq := strings.Join(f.Commands, "\n") - for _, want := range []string{"journalctl -u 'ob-sample-nightly.service'", "-t ob-run", "-o cat", "-r", "-n 20", "--no-pager"} { + for _, want := range []string{"journalctl SYSLOG_IDENTIFIER=ob-run ONEBOX_UNIT='ob-sample-nightly'", "-o cat", "-r", "-n 20", "--no-pager"} { if !strings.Contains(seq, want) { t.Fatalf("history read is missing %q:\n%s", want, seq) } @@ -87,7 +87,7 @@ func TestScheduleListReadsTimerState(t *testing.T) { func TestScheduleLogsTargetsOneInvocation(t *testing.T) { e, f := scheduledFixture(t) f.Dynamic = func(cmd string) (transport.Result, bool) { - if strings.Contains(cmd, "-t ob-run") { + if strings.Contains(cmd, "SYSLOG_IDENTIFIER=ob-run") { return transport.Result{Stdout: sampleRunRecords}, true } return transport.Result{}, false diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index f81c838..892863a 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -885,7 +885,8 @@ func TestScheduledJobNotifierWritesOneRunRecordToTheJournal(t *testing.T) { `"run":"%s","job":"%s","trigger":"%s","operation":"%s","release":"%s"`, `"duration_s":%s,"attempts":%s,"exit_status":%s,"outcome":"%s","inputs":{%s}`, `"${INVOCATION_ID:-}" 'nightly'`, - "systemd-cat -t ob-run", + `SYSLOG_IDENTIFIER=ob-run\nONEBOX_APP=%s\nONEBOX_UNIT=%s\nONEBOX_JOB=%s`, + `"$record" 'sample' 'ob-sample-nightly' 'nightly' | logger --journald`, } { if !strings.Contains(script, want) { t.Errorf("notifier is missing %q:\n%s", want, script) @@ -898,10 +899,10 @@ func TestScheduledJobNotifierWritesOneRunRecordToTheJournal(t *testing.T) { } } -// runNotifier executes the generated ExecStopPost script with stub systemd-cat -// and curl binaries, the way systemd would after a run. It returns the record -// the script wrote, whether the state file survived, and every curl -// invocation's arguments, one per element. +// runNotifier executes the generated ExecStopPost script with stub logger and +// curl binaries, the way systemd would after a run. It returns the record the +// script wrote, whether the state file survived, and every curl invocation's +// arguments, one per element. func runNotifier(t *testing.T, job app.ScheduledJob, notifications map[string]app.Notification, state string, env map[string]string) (map[string]any, bool, []string) { t.Helper() if runtime.GOOS == "windows" { @@ -928,8 +929,15 @@ func runNotifier(t *testing.T, job app.ScheduledJob, notifications map[string]ap } bin := t.TempDir() record := filepath.Join(bin, "record.jsonl") - stub := "#!/bin/sh\n[ \"$1\" = -t ] && [ \"$2\" = ob-run ] || exit 9\ncat >>" + record + "\n" - if err := os.WriteFile(filepath.Join(bin, "systemd-cat"), []byte(stub), 0o755); err != nil { + // The stub checks the structured fields the history query relies on and + // keeps only the MESSAGE line, as `journalctl -o cat` would show it. + stub := "#!/bin/sh\n[ \"$1\" = --journald ] || exit 9\n" + + "fields=$(cat)\n" + + "printf '%s\\n' \"$fields\" | grep -q '^SYSLOG_IDENTIFIER=ob-run$' || exit 8\n" + + "printf '%s\\n' \"$fields\" | grep -q '^ONEBOX_UNIT=ob-sample-nightly$' || exit 7\n" + + "printf '%s\\n' \"$fields\" | grep -q '^ONEBOX_JOB=nightly$' || exit 6\n" + + "printf '%s\\n' \"$fields\" | sed -n 's/^MESSAGE=//p' >>" + record + "\n" + if err := os.WriteFile(filepath.Join(bin, "logger"), []byte(stub), 0o755); err != nil { t.Fatal(err) } sent := filepath.Join(bin, "curl.args") diff --git a/site/src/content/docs/guides/schedule-a-job.mdx b/site/src/content/docs/guides/schedule-a-job.mdx index 922ebde..dfc8f25 100644 --- a/site/src/content/docs/guides/schedule-a-job.mdx +++ b/site/src/content/docs/guides/schedule-a-job.mdx @@ -132,7 +132,9 @@ Onebox when an application may not be deployed again soon. ## Every run leaves a record When a run ends, for any reason, the unit's `ExecStopPost` writes one record -to the host journal under the job's unit, with the syslog identifier `ob-run`: +to the host journal with the syslog identifier `ob-run` and the job's unit in +its `ONEBOX_UNIT` field, so `journalctl SYSLOG_IDENTIFIER=ob-run +ONEBOX_UNIT=ob-- -o cat` on the host is the raw history: ```json {"run":"a3f9…","job":"nightly-dump","trigger":"timer","operation":"","release":"20260905-140000-ab12cd","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:04:37Z","duration_s":276,"attempts":2,"exit_status":0,"outcome":"success","inputs":{}} diff --git a/site/src/content/docs/reference/cli.mdx b/site/src/content/docs/reference/cli.mdx index d1d7684..81871a2 100644 --- a/site/src/content/docs/reference/cli.mdx +++ b/site/src/content/docs/reference/cli.mdx @@ -981,7 +981,7 @@ Global Flags: ``` Read the run records the host wrote for one scheduled job. Each record is one activation: run id, trigger, release, start and end, attempts, exit status, outcome and, for a manual run, its inputs. -Records live in the host journal under the job's unit with syslog identifier ob-run; retention is the journal's. Reads only. +Records live in the host journal with syslog identifier ob-run and the job's unit in their ONEBOX_UNIT field; retention is the journal's. Reads only. Usage: ob schedule history [flags] From 40426798a38872129407c23c8552338b0294145c Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 16:25:01 -0700 Subject: [PATCH 16/25] refactor(schedule): the run record is the only verdict Drop the fallbacks: ob status no longer reads systemd's Result for an issue when no record exists, ob schedule logs no longer falls back to the unit log (and loses --tail), and ob schedule run --wait fails on any recorded outcome but success, a skip included. Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- cmd/ob/schedule.go | 8 ++--- internal/engine/schedule_history.go | 32 ++++++----------- internal/engine/schedule_history_test.go | 18 ++++++++-- internal/engine/schedule_run.go | 29 ++++++++------- internal/engine/schedule_run_test.go | 36 +++++++++++++++++++ internal/engine/schedule_status.go | 19 ++++------ internal/engine/schedule_test.go | 10 +++--- internal/engine/status_snapshot_test.go | 5 ++- .../content/docs/guides/schedule-a-job.mdx | 4 ++- site/src/content/docs/reference/cli.mdx | 1 - 10 files changed, 101 insertions(+), 61 deletions(-) diff --git a/cmd/ob/schedule.go b/cmd/ob/schedule.go index 8350075..cc90362 100644 --- a/cmd/ob/schedule.go +++ b/cmd/ob/schedule.go @@ -118,7 +118,6 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { scheduleCmd.AddCommand(historyCmd) var logsRun string - var logsTail int logsCmd := &cobra.Command{ Use: "logs ", Short: "journal of one scheduled run", @@ -136,7 +135,7 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { defer cleanup() if g.Output == "json" { var stdout, stderr bytes.Buffer - err = e.ScheduleLogs(cmd.Context(), args[0], logsRun, logsTail, &stdout, &stderr) + err = e.ScheduleLogs(cmd.Context(), args[0], logsRun, &stdout, &stderr) data := map[string]any{ "job": args[0], "run": logsRun, "stdout": stdout.String(), "stderr": stderr.String(), "passthrough_unredacted": true, @@ -153,7 +152,7 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { } if g.Output == "ndjson" { stream := newCLIRecordStream(cmd.OutOrStdout(), commandName(cmd)) - err = e.ScheduleLogs(cmd.Context(), args[0], logsRun, logsTail, stream.channelWriter("stdout"), stream.channelWriter("stderr")) + err = e.ScheduleLogs(cmd.Context(), args[0], logsRun, stream.channelWriter("stdout"), stream.channelWriter("stderr")) data := map[string]any{"job": args[0], "run": logsRun, "passthrough_unredacted": true} if err != nil { if writeErr := stream.terminal(cliOutcomeError, nil, publicError(err, "schedule_logs_failed", "run logs could not be read")); writeErr != nil { @@ -163,11 +162,10 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { } return stream.terminal(cliOutcomeSuccess, data, nil) } - return e.ScheduleLogs(cmd.Context(), args[0], logsRun, logsTail, cmd.OutOrStdout(), cmd.ErrOrStderr()) + return e.ScheduleLogs(cmd.Context(), args[0], logsRun, cmd.OutOrStdout(), cmd.ErrOrStderr()) }, } logsCmd.Flags().StringVar(&logsRun, "run", "", "run id from ob schedule history; default the newest run") - logsCmd.Flags().IntVarP(&logsTail, "tail", "n", 200, "lines to show when no run is recorded") scheduleCmd.AddCommand(logsCmd) var runInputs []string diff --git a/internal/engine/schedule_history.go b/internal/engine/schedule_history.go index 7483b5a..71c8c93 100644 --- a/internal/engine/schedule_history.go +++ b/internal/engine/schedule_history.go @@ -57,9 +57,8 @@ func scheduleHistoryCommand(unit string, n int) string { " -o cat -r -n " + strconv.Itoa(n) + " --no-pager 2>/dev/null || true" } -// parseScheduleRunRecords keeps the lines that decode and drops the rest. A -// host with a hand-edited unit or an older notifier may leave other text under -// the same identifier; one bad line must not hide the good ones. +// parseScheduleRunRecords keeps the lines that decode and drops the rest: a +// truncated or hand-written entry must not hide the records around it. func parseScheduleRunRecords(stdout string) []ScheduleRunRecord { var out []ScheduleRunRecord for _, line := range strings.Split(stdout, "\n") { @@ -153,34 +152,23 @@ func (e *Engine) ScheduleList(ctx context.Context) ([]ScheduleListing, error) { // ScheduleLogs streams the journal of one run. The run id is systemd's // invocation id, so the output is exactly that activation and nothing else. -// With no run given, the newest record's run is used; with no record at all, -// the unit's recent log stands in. -func (e *Engine) ScheduleLogs(ctx context.Context, name, run string, tail int, stdout, stderr io.Writer) error { - job, err := e.scheduledJob(name) - if err != nil { +// With no run given, the newest record's run is used. +func (e *Engine) ScheduleLogs(ctx context.Context, name, run string, stdout, stderr io.Writer) error { + if _, err := e.scheduledJob(name); err != nil { return err } - unit := e.names().ScheduledJobUnit(job.Name) if run == "" { records, err := e.ScheduleHistory(ctx, name, 1) if err != nil { return err } - if len(records) > 0 { - run = records[0].Run + if len(records) == 0 { + return fmt.Errorf("job %s has no recorded runs", name) } + run = records[0].Run } - if tail <= 0 { - tail = 200 - } - var cmd string - switch { - case run == "": - cmd = "journalctl -u " + q(unit+".service") + " -n " + strconv.Itoa(tail) + " --no-pager -o short-iso" - case scheduleRunID.MatchString(run): - cmd = "journalctl _SYSTEMD_INVOCATION_ID=" + run + " --no-pager -o short-iso" - default: + if !scheduleRunID.MatchString(run) { return fmt.Errorf("run id %q is not a systemd invocation id", run) } - return e.T.RunStream(ctx, cmd, stdout, stderr) + return e.T.RunStream(ctx, "journalctl _SYSTEMD_INVOCATION_ID="+run+" --no-pager -o short-iso", stdout, stderr) } diff --git a/internal/engine/schedule_history_test.go b/internal/engine/schedule_history_test.go index e038da2..665bfe7 100644 --- a/internal/engine/schedule_history_test.go +++ b/internal/engine/schedule_history_test.go @@ -93,14 +93,28 @@ func TestScheduleLogsTargetsOneInvocation(t *testing.T) { return transport.Result{}, false } var out bytes.Buffer - if err := e.ScheduleLogs(context.Background(), "nightly", "", 200, &out, &out); err != nil { + if err := e.ScheduleLogs(context.Background(), "nightly", "", &out, &out); err != nil { t.Fatal(err) } seq := strings.Join(f.Commands, "\n") if !strings.Contains(seq, "journalctl _SYSTEMD_INVOCATION_ID=b2c3d4e5f60718293a4b5c6d7e8f9012") { t.Fatalf("logs did not target the newest run:\n%s", seq) } - if err := e.ScheduleLogs(context.Background(), "nightly", "../etc", 200, &out, &out); err == nil { + if err := e.ScheduleLogs(context.Background(), "nightly", "../etc", &out, &out); err == nil { t.Fatal("an invalid run id reached the shell") } } + +func TestScheduleLogsRefusesAJobWithoutRecords(t *testing.T) { + e, f := scheduledFixture(t) + var out bytes.Buffer + err := e.ScheduleLogs(context.Background(), "nightly", "", &out, &out) + if err == nil || !strings.Contains(err.Error(), "no recorded runs") { + t.Fatalf("err = %v", err) + } + for _, command := range f.Commands { + if strings.Contains(command, "_SYSTEMD_INVOCATION_ID") || strings.Contains(command, "journalctl -u") { + t.Fatalf("logs guessed a source without a record: %s", command) + } + } +} diff --git a/internal/engine/schedule_run.go b/internal/engine/schedule_run.go index 122573e..406bdbf 100644 --- a/internal/engine/schedule_run.go +++ b/internal/engine/schedule_run.go @@ -133,19 +133,22 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu if err != nil { return result, err } - if len(records) > 0 { - result.Record = &records[0] - exit := "-" - if records[0].ExitStatus != nil { - exit = fmt.Sprint(*records[0].ExitStatus) - } - e.logf("schedule: %s run %s %s after %d attempt(s) in %ds (exit %s)", - name, records[0].Run, records[0].Outcome, records[0].Attempts, records[0].DurationSeconds, exit) - } - // A skipped run exits 75, which SuccessExitStatus makes a clean exit; any - // other non-zero exit is the job failing, and the record says how. - if res.ExitCode != 0 { - return result, fmt.Errorf("job %s did not succeed; see ob schedule logs %s", name, name) + if len(records) == 0 { + return result, fmt.Errorf("job %s ran (systemctl exit %d) but left no run record; the host's notifier did not write one", name, res.ExitCode) + } + last := records[0] + result.Record = &last + exit := "-" + if last.ExitStatus != nil { + exit = fmt.Sprint(*last.ExitStatus) + } + e.logf("schedule: %s run %s %s after %d attempt(s) in %ds (exit %s)", + name, last.Run, last.Outcome, last.Attempts, last.DurationSeconds, exit) + // The operator asked for this run and waited for it, so anything but a + // success is a failure of the request, a skip included: the unit exits 75 + // cleanly, but the work was not done. + if last.Outcome != "success" { + return result, fmt.Errorf("job %s run %s ended %s; see ob schedule logs %s --run %s", name, last.Run, last.Outcome, name, last.Run) } return result, nil } diff --git a/internal/engine/schedule_run_test.go b/internal/engine/schedule_run_test.go index 71855c2..572826d 100644 --- a/internal/engine/schedule_run_test.go +++ b/internal/engine/schedule_run_test.go @@ -113,3 +113,39 @@ func TestScheduleRunRefusals(t *testing.T) { t.Fatalf("active unit not refused: %v", err) } } + +func TestScheduleRunWaitReportsTheRecordAndFailsOnAnyOtherOutcome(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + for outcome, wantErr := range map[string]bool{"success": false, "skipped": true, "failure": true} { + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "systemctl start 'ob-sample-sync.service'"): + return transport.Result{}, true + case strings.Contains(cmd, "SYSLOG_IDENTIFIER=ob-run"): + return transport.Result{Stdout: `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"sync","trigger":"manual","started_at":"2026-09-05T15:00:01Z","finished_at":"2026-09-05T15:00:02Z","duration_s":1,"attempts":1,"exit_status":0,"outcome":"` + outcome + `","inputs":{}}` + "\n"}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + result, err := e.ScheduleRun(context.Background(), "op-"+outcome, "sync", nil, true) + if result.Record == nil || result.Record.Outcome != outcome { + t.Fatalf("%s: record not returned: %#v", outcome, result) + } + if (err != nil) != wantErr { + t.Fatalf("%s: err = %v, wantErr %v", outcome, err, wantErr) + } + if seq := strings.Join(f.Commands, "\n"); strings.Contains(seq, "--no-block") { + t.Fatalf("%s: --wait must block on systemctl start:\n%s", outcome, seq) + } + } +} diff --git a/internal/engine/schedule_status.go b/internal/engine/schedule_status.go index 24d38e0..8fa0c4d 100644 --- a/internal/engine/schedule_status.go +++ b/internal/engine/schedule_status.go @@ -11,10 +11,9 @@ import ( ) // StatusSchedule is the host-observed state of one declared scheduled job. -// systemd keeps Result after a oneshot exits, so a failed or timed-out run stays -// visible until a later successful run clears it. The run records the notifier -// writes to the journal say more: outcome, attempts, duration, and how many -// firings in a row have failed. +// systemd's Result and exit status are reported as observed; the verdict comes +// from the run records the notifier writes to the journal: outcome, attempts, +// duration, and how many firings in a row have failed. type StatusSchedule struct { Name string `json:"name"` Unit string `json:"unit"` @@ -186,19 +185,15 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) if service.loadState != "loaded" { status.Issues = append(status.Issues, "service unit is not loaded") } - switch { - case status.LastOutcome == "failure" || status.LastOutcome == "timeout": + // The record is the verdict. systemd's Result and ExecMainStatus are + // still reported as observed, but only a recorded failure or timeout + // is an issue. + if status.LastOutcome == "failure" || status.LastOutcome == "timeout" { exit := "?" if records[0].ExitStatus != nil { exit = strconv.Itoa(*records[0].ExitStatus) } status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %s)", status.LastOutcome, exit)) - case status.LastOutcome == "": - // No record yet: an older runner, or a journal that did not keep - // it. systemd's own result is the next best witness. - if service.result != "" && service.result != "success" { - status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %d)", service.result, service.exitStatus)) - } } status.Diverged = len(status.Issues) > 0 statuses = append(statuses, status) diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index 892863a..503cbc5 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -529,11 +529,13 @@ ActiveState=active if err != nil { t.Fatal(err) } - if len(statuses) != 1 || !statuses[0].Diverged || statuses[0].LastResult != "timeout" || statuses[0].LastExitStatus != 15 { - t.Fatalf("failed systemd result was not surfaced: %#v", statuses) + // systemd's own result is reported, but the verdict is the run record, + // and there is none here. + if len(statuses) != 1 || statuses[0].LastResult != "timeout" || statuses[0].LastExitStatus != 15 { + t.Fatalf("systemd result was not surfaced: %#v", statuses) } - if !strings.Contains(strings.Join(statuses[0].Issues, "\n"), "last run failed") { - t.Fatalf("failure has no actionable issue: %#v", statuses[0]) + if statuses[0].Diverged || len(statuses[0].Issues) != 0 { + t.Fatalf("an issue was raised without a run record: %#v", statuses[0]) } } diff --git a/internal/engine/status_snapshot_test.go b/internal/engine/status_snapshot_test.go index 0d3fe1c..0d053a3 100644 --- a/internal/engine/status_snapshot_test.go +++ b/internal/engine/status_snapshot_test.go @@ -248,6 +248,9 @@ ExecMainStatus=9 @@nightly:timer LoadState=loaded ActiveState=active +@@nightly:run +@@nightly:history +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:09Z","duration_s":8,"attempts":1,"exit_status":9,"outcome":"failure","inputs":{}} `}, true } return base(cmd) @@ -260,7 +263,7 @@ ActiveState=active if !snapshot.Complete || !snapshot.Diverged || len(snapshot.Schedules) != 1 { t.Fatalf("scheduled failure was not included as observed divergence: %#v", snapshot) } - if got := snapshot.Schedules[0]; !got.Diverged || got.LastResult != "exit-code" || got.LastExitStatus != 9 { + if got := snapshot.Schedules[0]; !got.Diverged || got.LastOutcome != "failure" || got.LastResult != "exit-code" || got.LastExitStatus != 9 { t.Fatalf("unexpected scheduled-job status: %#v", got) } } diff --git a/site/src/content/docs/guides/schedule-a-job.mdx b/site/src/content/docs/guides/schedule-a-job.mdx index dfc8f25..08359e7 100644 --- a/site/src/content/docs/guides/schedule-a-job.mdx +++ b/site/src/content/docs/guides/schedule-a-job.mdx @@ -269,7 +269,9 @@ gates of `ob job run` below. The runner tells a manual activation from a timer firing by the `TRIGGER_UNIT` variable systemd sets on timer activations, which needs systemd 252 or newer on the host. Ubuntu 24.04 and Debian 12 qualify; a job that declares inputs is -refused at deploy time on an older host. +refused at deploy time on an older host. A job without inputs still runs on an +older host, but its records say `manual` for every activation, because nothing +there tells the runner otherwise. ## Running one by hand diff --git a/site/src/content/docs/reference/cli.mdx b/site/src/content/docs/reference/cli.mdx index 81871a2..349f577 100644 --- a/site/src/content/docs/reference/cli.mdx +++ b/site/src/content/docs/reference/cli.mdx @@ -1029,7 +1029,6 @@ Usage: Flags: -h, --help help for logs --run string run id from ob schedule history; default the newest run - -n, --tail int lines to show when no run is recorded (default 200) Global Flags: -c, --config string path to the project YAML file (default "ob.yml") From 48cc3f148a2c4df353f6e13ee1e2f5cb569982f3 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 16:59:25 -0700 Subject: [PATCH 17/25] fix(schedule): act on the local review - A skip is the runner's own word: it records the reason and exits 0 before any container starts, the unit keeps no SuccessExitStatus remap, a job that exits 75 is a failure, and three skips in a row are an ob status issue. The application lock is honoured only within its TTL, as AcquireLock does. - Every scheduled job needs systemd 252 (TRIGGER_UNIT), checked in preflight before staging as well as in schedule reconciliation. - ob schedule run --wait matches the record by operation id, polls briefly for journald, and discards its inputs file when the start fails. - The retry validator counts the same whole seconds the runner sleeps. - The published schema constrains notify, retry, inputs and job-only fields. - ob schedule logs reports the run id it resolved; ob status drops systemd's Result in favour of the record alone. Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- cmd/ob/schedule.go | 11 +- docs/onebox.run-v1.schema.json | 37 ++++- e2e/server_test.go | 55 ++++++- e2e/testdata/postgres/ob.yml.tmpl | 30 +++- internal/app/jsonschema.go | 13 +- internal/app/schedule_retry.go | 29 ++-- internal/engine/deploy_test.go | 4 + internal/engine/preflight.go | 9 ++ internal/engine/schedule.go | 153 ++++++++++-------- internal/engine/schedule_history.go | 45 +++--- internal/engine/schedule_history_test.go | 10 +- internal/engine/schedule_run.go | 63 ++++++-- internal/engine/schedule_run_test.go | 40 ++++- internal/engine/schedule_status.go | 66 ++++---- internal/engine/schedule_test.go | 144 +++++++++-------- internal/engine/status.go | 10 +- internal/engine/status_snapshot_test.go | 2 +- site/public/onebox.run-v1.schema.json | 37 ++++- .../content/docs/guides/schedule-a-job.mdx | 26 +-- .../docs/reference/fields/workloads.mdx | 8 +- 20 files changed, 553 insertions(+), 239 deletions(-) diff --git a/cmd/ob/schedule.go b/cmd/ob/schedule.go index cc90362..e440223 100644 --- a/cmd/ob/schedule.go +++ b/cmd/ob/schedule.go @@ -135,9 +135,9 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { defer cleanup() if g.Output == "json" { var stdout, stderr bytes.Buffer - err = e.ScheduleLogs(cmd.Context(), args[0], logsRun, &stdout, &stderr) + run, err := e.ScheduleLogs(cmd.Context(), args[0], logsRun, &stdout, &stderr) data := map[string]any{ - "job": args[0], "run": logsRun, "stdout": stdout.String(), "stderr": stderr.String(), + "job": args[0], "run": run, "stdout": stdout.String(), "stderr": stderr.String(), "passthrough_unredacted": true, } if err != nil { @@ -152,8 +152,8 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { } if g.Output == "ndjson" { stream := newCLIRecordStream(cmd.OutOrStdout(), commandName(cmd)) - err = e.ScheduleLogs(cmd.Context(), args[0], logsRun, stream.channelWriter("stdout"), stream.channelWriter("stderr")) - data := map[string]any{"job": args[0], "run": logsRun, "passthrough_unredacted": true} + run, err := e.ScheduleLogs(cmd.Context(), args[0], logsRun, stream.channelWriter("stdout"), stream.channelWriter("stderr")) + data := map[string]any{"job": args[0], "run": run, "passthrough_unredacted": true} if err != nil { if writeErr := stream.terminal(cliOutcomeError, nil, publicError(err, "schedule_logs_failed", "run logs could not be read")); writeErr != nil { return writeErr @@ -162,7 +162,8 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { } return stream.terminal(cliOutcomeSuccess, data, nil) } - return e.ScheduleLogs(cmd.Context(), args[0], logsRun, cmd.OutOrStdout(), cmd.ErrOrStderr()) + _, err = e.ScheduleLogs(cmd.Context(), args[0], logsRun, cmd.OutOrStdout(), cmd.ErrOrStderr()) + return err }, } logsCmd.Flags().StringVar(&logsRun, "run", "", "run id from ob schedule history; default the newest run") diff --git a/docs/onebox.run-v1.schema.json b/docs/onebox.run-v1.schema.json index 4344bd9..76ba0e3 100644 --- a/docs/onebox.run-v1.schema.json +++ b/docs/onebox.run-v1.schema.json @@ -1878,6 +1878,11 @@ "required": [ "schedule" ] + }, + { + "required": [ + "inputs" + ] } ] } @@ -2234,6 +2239,18 @@ "inputs": { "additionalProperties": { "additionalProperties": false, + "oneOf": [ + { + "required": [ + "enum" + ] + }, + { + "required": [ + "pattern" + ] + } + ], "patternProperties": { "^x-": {} }, @@ -2264,9 +2281,15 @@ "type": "string" } }, + "required": [ + "default" + ], "type": "object" }, "description": "Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them.", + "propertyNames": { + "pattern": "^[A-Z][A-Z0-9_]*$" + }, "type": "object" }, "labels": { @@ -2578,6 +2601,12 @@ "default": "failure, timeout", "description": "Run outcomes that send the configured notifications: success, failure, timeout, skipped.", "items": { + "enum": [ + "success", + "failure", + "timeout", + "skipped" + ], "type": "string" }, "type": "array" @@ -2595,22 +2624,26 @@ "examples": [ 3 ], + "maximum": 10, + "minimum": 1, "type": "integer" }, "backoff": { "default": "30s", - "description": "Sleep before the second attempt; it doubles after each failure.", + "description": "Sleep before the second attempt; it doubles after each failure. Expects a duration such as 30s, 5m, 1h30m or 14d.", "examples": [ "1m" ], + "pattern": "^(([0-9]+([.][0-9]+)?(ns|us|µs|ms|s|m|h))+|[0-9]+d)$", "type": "string" }, "max_backoff": { "default": "10m", - "description": "Upper bound for the doubling sleep.", + "description": "Upper bound for the doubling sleep. Expects a duration such as 30s, 5m, 1h30m or 14d.", "examples": [ "30m" ], + "pattern": "^(([0-9]+([.][0-9]+)?(ns|us|µs|ms|s|m|h))+|[0-9]+d)$", "type": "string" } }, diff --git a/e2e/server_test.go b/e2e/server_test.go index 877fee8..8637510 100644 --- a/e2e/server_test.go +++ b/e2e/server_test.go @@ -236,6 +236,46 @@ ExecStart=/usr/bin/docker compose -p observer -f /var/lib/ob/observer/current/co t.Fatalf("normal scheduled run result = %q, want success", result) } + // The notifier wrote a record for that run; a hand-started unit has no + // TRIGGER_UNIT, so it is recorded as a manual activation. + history := s.mustOb(t, dir, "schedule", "history", "chore", "--output", "json") + for _, want := range []string{`"outcome": "success"`, `"trigger": "manual"`, `"attempts": 1`} { + if !strings.Contains(history, want) { + t.Fatalf("run history is missing %q:\n%s", want, history) + } + } + + // One failure, one sleep, one success: the record counts both attempts. + s.run(t, "rm -rf /tmp/onebox-e2e-retry && mkdir -p /tmp/onebox-e2e-retry") + s.run(t, "systemctl start ob-observer-retry--chore.service") + retry := s.mustOb(t, dir, "schedule", "history", "retry-chore", "--output", "json") + for _, want := range []string{`"outcome": "success"`, `"attempts": 2`} { + if !strings.Contains(retry, want) { + t.Fatalf("retry history is missing %q:\n%s", want, retry) + } + } + + // A manual run with an input override reaches the container as its + // environment, is journaled with the operator, and shows up as such. + manual := s.mustOb(t, dir, "schedule", "run", "input-chore", "--input", "GREETING=hello", "--wait", "--output", "json") + for _, want := range []string{`"GREETING": "hello"`, `"outcome": "success"`, `"trigger": "manual"`} { + if !strings.Contains(manual, want) { + t.Fatalf("manual run result is missing %q:\n%s", want, manual) + } + } + logs := s.mustOb(t, dir, "schedule", "logs", "input-chore") + if !strings.Contains(logs, "greeting=hello") { + t.Fatalf("run logs do not show the override:\n%s", logs) + } + audit := s.mustOb(t, dir, "audit") + if !strings.Contains(audit, "schedule run") { + t.Fatalf("audit does not list the manual run:\n%s", audit) + } + list := s.mustOb(t, dir, "schedule", "list") + if !strings.Contains(list, "input-chore") || !strings.Contains(list, "active") { + t.Fatalf("schedule list did not show the timer:\n%s", list) + } + // The receiver lives on the target because host-fired notifications do // too. It accepts one POST, records the body, and exits. receiver := `from http.server import BaseHTTPRequestHandler, HTTPServer @@ -271,6 +311,10 @@ HTTPServer(("127.0.0.1", 18080), Handler).handle_request() "systemctl show ob-observer-timeout--chore.service --property=Result --value")); result != "timeout" { t.Fatalf("timed-out scheduled run result = %q, want timeout", result) } + // The runner was killed mid-run; ExecStopPost still wrote the record. + if timedOut := s.mustOb(t, dir, "schedule", "history", "timeout-chore", "--output", "json"); !strings.Contains(timedOut, `"outcome": "timeout"`) { + t.Fatalf("timeout was not recorded:\n%s", timedOut) + } out, err := s.ob(t, dir, "status") if err == nil || !strings.Contains(out, "schedule timeout-chore") || !strings.Contains(out, "last run failed: timeout") { t.Fatalf("status did not expose the scheduled failure (err=%v):\n%s", err, out) @@ -286,9 +330,14 @@ HTTPServer(("127.0.0.1", 18080), Handler).handle_request() t.Fatalf("scheduled failure notification is missing %q: %s", want, notification) } } - // Do not make the deliberately induced failure pollute later lifecycle - // assertions; systemd resets Result to success with the failed state. - s.run(t, "systemctl reset-failed ob-observer-timeout--chore.service") + // The record is the verdict, so `systemctl reset-failed` no longer + // clears the failure from `ob status`; only a later successful run + // does. A manual run with the input that makes the job finish in time + // is that run, and it must leave status green for the steps after. + s.mustOb(t, dir, "schedule", "run", "timeout-chore", "--input", "SLEEP=0", "--wait") + if cleared := s.mustOb(t, dir, "status"); !strings.Contains(cleared, "schedule timeout-chore active") { + t.Fatalf("a successful manual run did not clear the recorded timeout:\n%s", cleared) + } }) t.Run("preflight", func(t *testing.T) { diff --git a/e2e/testdata/postgres/ob.yml.tmpl b/e2e/testdata/postgres/ob.yml.tmpl index f22ba03..000d834 100644 --- a/e2e/testdata/postgres/ob.yml.tmpl +++ b/e2e/testdata/postgres/ob.yml.tmpl @@ -41,14 +41,38 @@ workloads: data_effect: none schedule: { cron: "0 0 1 1 *", timeout: 20s, catch_up: false } # A deliberately wedged timer run proves the host-enforced timeout becomes a - # failed systemd result and that `ob status` exposes it. Its annual timer never - # fires during the suite; the test starts the service directly. + # recorded timeout that `ob status` exposes. Its annual timer never fires + # during the suite; the test starts the service directly. The SLEEP input + # lets a later manual run succeed, which is the only thing that clears a + # recorded failure. timeout-chore: role: job image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 - command: ["sh", "-c", "sleep 30"] + command: ["sh", "-c", "sleep $SLEEP"] data_effect: none + inputs: + SLEEP: {enum: ["0", "30"], default: "30"} schedule: { cron: "0 0 1 1 *", timeout: 1s, catch_up: false } + # Fails once, then succeeds: proves the in-firing retry and that the run + # record counts attempts. The marker lives on the host so the second + # attempt, a fresh container, can see the first one ran. + retry-chore: + role: job + image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 + command: ["sh", "-c", "if [ -f /marker/ran ]; then echo second; else touch /marker/ran; exit 1; fi"] + data_effect: none + volumes: [{source: /tmp/onebox-e2e-retry, path: /marker}] + schedule: { cron: "0 0 1 1 *", timeout: 60s, catch_up: false, retry: {attempts: 2, backoff: 1s} } + # A declared input reaches the container as an environment variable: its + # default on a timer-shaped start, an override on a manual run. + input-chore: + role: job + image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 + command: ["sh", "-c", "echo greeting=$GREETING"] + data_effect: none + inputs: + GREETING: {enum: [hi, hello], default: hi} + schedule: { cron: "0 0 1 1 *", timeout: 20s, catch_up: false } deployment: order: [app] services: diff --git a/internal/app/jsonschema.go b/internal/app/jsonschema.go index ebc55a6..920d4f1 100644 --- a/internal/app/jsonschema.go +++ b/internal/app/jsonschema.go @@ -343,6 +343,17 @@ var schemaConstraints = []struct { {[]string{"workloads", "*", "drain", "signal"}, pattern(gSignal)}, {[]string{"workloads", "*", "drain", "wait"}, pattern(gDur)}, {[]string{"workloads", "*", "drain", "grace"}, pattern(gDur)}, + {[]string{"workloads", "*", "schedule", "notify", "items"}, enum(eScheduleNotify)}, + {[]string{"workloads", "*", "schedule", "retry", "attempts"}, map[string]any{"minimum": 1, "maximum": maxRetryAttempts}}, + {[]string{"workloads", "*", "schedule", "retry", "backoff"}, pattern(gDur)}, + {[]string{"workloads", "*", "schedule", "retry", "max_backoff"}, pattern(gDur)}, + {[]string{"workloads", "*", "inputs"}, propertyNames(gInputName)}, + // An input says what it accepts, one way, and always has a default: the + // timer fires without anyone to ask. + {[]string{"workloads", "*", "inputs", "*"}, map[string]any{ + "required": []any{"default"}, + "oneOf": anyRequired([]any{"enum", "pattern"}), + }}, {[]string{"workloads", "*", "resources", "memory"}, pattern(gSize)}, {[]string{"workloads", "*", "resources", "cpus"}, pattern(gCpus)}, {[]string{"workloads", "*", "persistence", "mode"}, enum(ePersistence)}, @@ -494,7 +505,7 @@ func applyRoleRules(doc map[string]any) { map[string]any{"anyOf": anyRequired(sources)}, } - jobOnly := []any{"when", "data_effect", "schedule"} + jobOnly := []any{"when", "data_effect", "schedule", "inputs"} workload["allOf"] = []any{ // Exactly one source. A workload with none cannot run and a workload // with two does not say which image it is. diff --git a/internal/app/schedule_retry.go b/internal/app/schedule_retry.go index c6e69b5..2c728b8 100644 --- a/internal/app/schedule_retry.go +++ b/internal/app/schedule_retry.go @@ -1,6 +1,9 @@ package app -import "time" +import ( + "math" + "time" +) // The retry defaults reproduce today's behaviour exactly: one attempt, and the // backoff values are inert until attempts rises above one. @@ -15,20 +18,28 @@ const ( // failure and timeout, stay quiet on success, and never mention a skip. var defaultScheduleNotify = []string{"failure", "timeout"} +// RetryBackoffSeconds is the whole-second form the runner sleeps: `sleep` +// takes seconds, and a fraction rounds up rather than down to a busy loop. +func RetryBackoffSeconds(d time.Duration) int { + return int(math.Ceil(d.Seconds())) +} + // scheduleRetryWorstCase is the longest a run can spend asleep between -// attempts: the doubling series, each term capped, over the attempts-1 sleeps. -// Validation keeps it under the timeout so the last attempt can always start. +// attempts. It reproduces the runner's own arithmetic step for step: whole +// seconds, sleep, double, cap, over the attempts-1 sleeps. Validation keeps it +// under the timeout so the last attempt can always start, and that promise +// only holds if both sides count the same way. func scheduleRetryWorstCase(attempts int, backoff, max time.Duration) time.Duration { - var total time.Duration - sleep := backoff + total := 0 + sleep, cap := RetryBackoffSeconds(backoff), RetryBackoffSeconds(max) for i := 1; i < attempts; i++ { - if sleep > max { - sleep = max - } total += sleep sleep *= 2 + if sleep > cap { + sleep = cap + } } - return total + return time.Duration(total) * time.Second } // retryPolicy resolves the declared block over the defaults. Unparseable diff --git a/internal/engine/deploy_test.go b/internal/engine/deploy_test.go index bf52fb9..7a23f28 100644 --- a/internal/engine/deploy_test.go +++ b/internal/engine/deploy_test.go @@ -59,6 +59,10 @@ func happyFake() *transport.Fake { if strings.Contains(cmd, "Config.Healthcheck") { return transport.Result{Stdout: `{"Test":` + guardedHealthcheck + `,"Interval":5000000000,"Retries":3}` + "\n"}, true } + // The host scheduled jobs need: systemd 252 or newer. + if strings.Contains(cmd, "systemctl --version") { + return transport.Result{Stdout: "systemd 255 (255.4-1ubuntu8)\n"}, true + } // server roll state, derived from history so the loop converges: NEW1 // appears after a scale, OLD1 disappears once removed, names track renames. scaled, oldGone, drained := false, false, false diff --git a/internal/engine/preflight.go b/internal/engine/preflight.go index a78a790..6e6b689 100644 --- a/internal/engine/preflight.go +++ b/internal/engine/preflight.go @@ -37,6 +37,15 @@ func (e *Engine) preflight(ctx context.Context, requireDiscovery bool) error { } return app.HostPrerequisiteRefusal("%s: %s", e.T.Host(), unmet.Message) } + // Scheduled jobs have host requirements of their own. Asked here so a + // deploy refuses before staging a release, not after activating it. + jobs, err := e.Spec.ScheduledJobs() + if err != nil { + return err + } + if err := e.requireScheduleHost(ctx, jobs); err != nil { + return err + } base := release.PathsFor(e.names()).Base if res, err := e.T.Run(ctx, "mkdir -p "+q(base)+" && test -w "+q(base)); err != nil || res.ExitCode != 0 { return fmt.Errorf("%s not writable by deploy user", base) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 7588604..0aecac8 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -8,6 +8,7 @@ import ( "regexp" "strconv" "strings" + "time" "github.com/labstack/onebox/internal/app" "github.com/labstack/onebox/internal/notify" @@ -81,22 +82,8 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { } wanted := map[string]bool{} - if len(jobs) > 0 && !e.hasFlock(ctx) { - return errors.New("scheduled jobs require flock on the target so they cannot overlap deployments; install util-linux and deploy again") - } - if needsTriggerUnit(jobs) { - // A manual activation is told apart from a timer firing by the - // TRIGGER_UNIT variable systemd 252 introduced. Without it the runner - // could not know whether to read the inputs file, so inputs stay - // refused on an older host rather than guessed at. - res, err := e.T.Run(ctx, "systemctl --version 2>/dev/null | head -1") - if err != nil { - return err - } - if version, ok := systemdVersion(res.Stdout); !ok || version < 252 { - return fmt.Errorf("a job declares inputs, which need systemd 252 or newer on the host for $TRIGGER_UNIT; the host reports %q", - strings.TrimSpace(res.Stdout)) - } + if err := e.requireScheduleHost(ctx, jobs); err != nil { + return err } for _, job := range jobs { unit := n.ScheduledJobUnit(job.Name) @@ -122,7 +109,7 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { if e.Spec.Runtime != nil { runtimeEnvFiles = e.Spec.Runtime.EnvFiles } - runner := scheduleRunnerScript(e.Spec.Name, job, n, e.lockPath(), runtimeEnvFiles) + runner := scheduleRunnerScript(e.Spec.Name, job, n, e.lockPath(), runtimeEnvFiles, e.lockTTL()) notifier, err := e.scheduleNotifier(job) if err != nil { return fmt.Errorf("job %s: cannot render its failure notifier: %w", job.Name, err) @@ -185,9 +172,9 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { // job explicitly opts into the narrower pinned-release contract. Pinned mode // meets the deploy acquirer briefly under schedule.lock, leases the resolved // release before releasing that rendezvous, then retains only its own job lock. -func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile) string { +func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile, lockTTL time.Duration) string { if job.DeployLock == "pinned" { - return pinnedScheduleRunnerScript(application, job, names, applicationLock, runtimeEnvFiles) + return pinnedScheduleRunnerScript(application, job, names, applicationLock, runtimeEnvFiles, lockTTL) } container := names.Container(job.Name, 1) projectDir := q(names.CurrentLink()) @@ -201,32 +188,27 @@ func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Na "install -d -m 700 " + q(names.AppDir()+"/schedule"), } lines = append(lines, scheduleInputsLines(names.ScheduledJobRunInputs(job.Name))...) + lines = append(lines, scheduleLockLines(names, job.Name, applicationLock, lockTTL)...) lines = append(lines, - "exec 9>"+q(names.ScheduledJobRunLock(job.Name)), - "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 9", - "exec 8>"+q(names.ScheduleRunLock()), - "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 8", - "if [ -e "+q(applicationLock)+" ]; then echo 'onebox: an application operation holds the deploy lock' >&2; exit 75; fi", // Best effort: the record names the release that ran, and an exclusive // job runs whatever `current` points at when it starts. "release_dir=$(readlink -f "+q(names.CurrentLink())+" 2>/dev/null || true)", "release=${release_dir##*/}", scheduleContainerCleanup(container), - "cleanup() { "+scheduleContainerCleanup(container)+"; [ -z \"${tmp:-}\" ] || rm -f \"$tmp\"; }", + "cleanup() { "+scheduleContainerCleanup(container)+"; rm -f \"$tmp\"; }", "trap cleanup 0", "trap 'exit 129' 1", "trap 'exit 130' 2", "trap 'exit 143' 15", ) - lines = append(lines, scheduleRunPreamble(names.ScheduledJobRunState(job.Name))...) + lines = append(lines, scheduleRunPreamble()...) lines = append(lines, scheduleAttemptLoop(job, compose)...) lines = append(lines, "") return strings.Join(lines, "\n") } -func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile) string { +func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile, lockTTL time.Duration) string { scheduleDir := names.AppDir() + "/schedule" - state := names.ScheduledJobRunState(job.Name) container := names.Container(job.Name, 1) projectDir := `"$release_dir"` compose := "/usr/bin/docker compose -p " + q(application) + " --project-directory " + projectDir + @@ -239,28 +221,25 @@ func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names "install -d -m 700 " + q(scheduleDir), } lines = append(lines, scheduleInputsLines(names.ScheduledJobRunInputs(job.Name))...) + lines = append(lines, scheduleLockLines(names, job.Name, applicationLock, lockTTL)...) lines = append(lines, - "exec 9>"+q(names.ScheduledJobRunLock(job.Name)), - "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 9", - "exec 8>"+q(names.ScheduleRunLock()), - "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 8", - "if [ -e "+q(applicationLock)+" ]; then echo 'onebox: an application operation holds the deploy lock' >&2; exit 75; fi", - "release_dir=$(readlink -f "+q(names.CurrentLink())+") || { echo 'onebox: current release cannot be resolved' >&2; exit 75; }", - "if [ \"${release_dir%/*}\" != "+q(names.ReleasesDir())+" ]; then echo 'onebox: current release resolves outside the release store' >&2; exit 75; fi", + // These are misconfigurations, not timing: the run fails, loudly. + "release_dir=$(readlink -f "+q(names.CurrentLink())+") || { echo 'onebox: current release cannot be resolved' >&2; exit 1; }", + "if [ \"${release_dir%/*}\" != "+q(names.ReleasesDir())+" ]; then echo 'onebox: current release resolves outside the release store' >&2; exit 1; fi", "release=${release_dir##*/}", - "if ! printf '%s\\n' \"$release\" | grep -Eq '^[0-9]{8}-[0-9]{6}-[0-9A-Za-z_-]+$'; then echo 'onebox: current release identity is invalid' >&2; exit 75; fi", - "if [ ! -f \"$release_dir/compose.yaml\" ]; then echo 'onebox: pinned release has no compose.yaml' >&2; exit 75; fi", + "if ! printf '%s\\n' \"$release\" | grep -Eq '^[0-9]{8}-[0-9]{6}-[0-9A-Za-z_-]+$'; then echo 'onebox: current release identity is invalid' >&2; exit 1; fi", + "if [ ! -f \"$release_dir/compose.yaml\" ]; then echo 'onebox: pinned release has no compose.yaml' >&2; exit 1; fi", "exec 7>>\"$release_dir/.ob-schedule.lease\"", "chmod 600 \"$release_dir/.ob-schedule.lease\"", "/usr/bin/flock --shared 7", scheduleContainerCleanup(container), - "cleanup() { "+scheduleContainerCleanup(container)+"; [ -z \"${tmp:-}\" ] || rm -f \"$tmp\"; }", + "cleanup() { "+scheduleContainerCleanup(container)+"; rm -f \"$tmp\"; }", "trap cleanup 0", "trap 'exit 129' 1", "trap 'exit 130' 2", "trap 'exit 143' 15", ) - lines = append(lines, scheduleRunPreamble(state)...) + lines = append(lines, scheduleRunPreamble()...) // The lease is held; the schedule mutex goes back before the first // attempt so a compatible deploy is not blocked through the backoff. lines = append(lines, "/usr/bin/flock --unlock 8") @@ -269,6 +248,63 @@ func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names return strings.Join(lines, "\n") } +// scheduleLockLines take the run's locks, and turn a conflict into a recorded +// skip rather than a failed unit. A skip is a fact about timing: another run +// of this job, or an application operation, is in progress. The runner writes +// the reason into the state file and exits 0, so systemd sees a clean unit and +// the notifier records `skipped` with that reason. The container's own exit +// status is never mistaken for a skip, because a skip happens before any +// container starts. +// +// The application lock is honoured for as long as AcquireLock would honour it: +// a lock older than the TTL belongs to a runner that died, and AcquireLock +// takes it over, so the timer must not defer to it forever either. +func scheduleLockLines(names app.Names, job, applicationLock string, lockTTL time.Duration) []string { + ttlMinutes := int(math.Ceil(lockTTL.Minutes())) + if ttlMinutes < 1 { + ttlMinutes = 1 + } + return []string{ + "state=" + q(names.ScheduledJobRunState(job)), + "tmp=\"$state.$$\"", + // The operation and inputs of a manual request are kept on the skip + // record too, so `ob schedule run --wait` can find its own outcome. + "skip() { umask 077; printf 'skipped=%s\\noperation=%s\\ninputs=%s\\n' \"$1\" \"$operation\" \"$inputs_json\" >\"$tmp\"; mv -f \"$tmp\" \"$state\"; echo \"onebox: skipped: $1\" >&2; exit 0; }", + "exec 9>" + q(names.ScheduledJobRunLock(job)), + "/usr/bin/flock --exclusive --nonblock 9 || skip 'another run of this job is still in progress'", + "exec 8>" + q(names.ScheduleRunLock()), + "/usr/bin/flock --exclusive --nonblock 8 || skip 'an application operation is taking its lock'", + "if [ -e " + q(applicationLock) + " ] && [ -z \"$(find " + q(applicationLock) + " -mmin +" + strconv.Itoa(ttlMinutes) + " 2>/dev/null)\" ]; then skip 'an application operation holds the deploy lock'; fi", + } +} + +// requireScheduleHost is what a host needs before any scheduled job can be +// installed on it. Preflight asks it so a deploy refuses before staging, and +// SyncSchedules asks again so `ob schedule apply` cannot bypass it. +// +// systemd 252 introduced TRIGGER_UNIT, which is how the runner tells a timer +// firing from an operator's start. On an older systemd every activation would +// look manual: recorded as such, and consuming a pending inputs file that was +// meant for the operator's run. The floor applies to every scheduled job, not +// only those with inputs, because the record's trigger is part of the contract. +func (e *Engine) requireScheduleHost(ctx context.Context, jobs []app.ScheduledJob) error { + if len(jobs) == 0 { + return nil + } + if !e.hasFlock(ctx) { + return errors.New("scheduled jobs require flock on the target so they cannot overlap deployments; install util-linux and deploy again") + } + res, err := e.T.Run(ctx, "systemctl --version 2>/dev/null | head -1") + if err != nil { + return err + } + if version, ok := systemdVersion(res.Stdout); !ok || version < 252 { + return fmt.Errorf("scheduled jobs need systemd 252 or newer on the host, which tells a timer firing from a manual start; the host reports %q", + strings.TrimSpace(res.Stdout)) + } + return nil +} + func scheduleContainerCleanup(container string) string { return "/usr/bin/docker rm -f " + q(container) + " >/dev/null 2>&1 || true" } @@ -291,10 +327,8 @@ func scheduleStateFunction() []string { // scheduleRunPreamble sets the variables write_state records. The trigger is // systemd's own word for it: a timer activation carries TRIGGER_UNIT (systemd // 252 and newer), anything else is an operator. -func scheduleRunPreamble(state string) []string { +func scheduleRunPreamble() []string { return append([]string{ - "state=" + q(state), - "tmp=\"$state.$$\"", "started_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", "started_epoch=$(date -u '+%s')", "if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=manual; fi", @@ -323,15 +357,6 @@ func scheduleInputsLines(inputsPath string) []string { } } -func needsTriggerUnit(jobs []app.ScheduledJob) bool { - for _, job := range jobs { - if len(job.Inputs) > 0 { - return true - } - } - return false -} - // systemdVersion reads the leading number from `systemd 255 (255.4-1ubuntu8)`. func systemdVersion(firstLine string) (int, bool) { fields := strings.Fields(firstLine) @@ -352,8 +377,9 @@ func scheduleAttemptLoop(job app.ScheduledJob, compose string) []string { } return []string{ fmt.Sprintf("max_attempts=%d", job.RetryAttempts), - fmt.Sprintf("backoff=%d", int(math.Ceil(job.RetryBackoff.Seconds()))), - fmt.Sprintf("max_backoff=%d", int(math.Ceil(job.RetryMaxBackoff.Seconds()))), + // Whole seconds, the same rounding validation used to bound the sum. + fmt.Sprintf("backoff=%d", app.RetryBackoffSeconds(job.RetryBackoff)), + fmt.Sprintf("max_backoff=%d", app.RetryBackoffSeconds(job.RetryMaxBackoff)), "attempt=1", "while :; do", " write_state \"$attempt\"", @@ -392,15 +418,11 @@ func scheduleServiceUnit(application string, job app.ScheduledJob, runnerPath, n "[Service]", "Type=oneshot", "ExecStart=/bin/sh " + runnerPath, - // ExecStopPost runs after success, start failures, and timeouts. It always - // attempts fenced container cleanup, then uses SERVICE_RESULT to decide - // whether failure notifications are needed. + // ExecStopPost runs after success, failures, and timeouts. It always + // attempts fenced container cleanup, writes the run record from the + // runner's state and SERVICE_RESULT, then notifies per the job's policy. "ExecStopPost=/bin/sh " + notifyPath, "TimeoutStartSec=" + job.Timeout, - // Exit 75 is the runner's "skipped for a lock conflict". It is a fact - // about timing, not a failure of the job, and it must not leave the - // unit failed or trip failure notifications. - "SuccessExitStatus=75", "", }, "\n") } @@ -428,7 +450,7 @@ const scheduleRunIdentifier = "ob-run" func scheduleRunRecordLines(application, unit, job, state string) []string { return []string{ "state=" + q(state), - "release=''; started_at=''; started_epoch=''; trigger=''; operation=''; attempt=0; inputs=''", + "release=''; started_at=''; started_epoch=''; trigger=''; operation=''; attempt=0; inputs=''; skipped=''", "if [ -f \"$state\" ]; then", " while IFS= read -r line || [ -n \"$line\" ]; do", " case \"$line\" in", @@ -439,6 +461,7 @@ func scheduleRunRecordLines(application, unit, job, state string) []string { " operation=*) operation=${line#operation=} ;;", " attempt=*) attempt=${line#attempt=} ;;", " inputs=*) inputs=${line#inputs=} ;;", + " skipped=*) skipped=${line#skipped=} ;;", " esac", " done <\"$state\"", " rm -f \"$state\"", @@ -449,8 +472,10 @@ func scheduleRunRecordLines(application, unit, job, state string) []string { // EXIT_STATUS is a signal name when the main process was killed. "case \"$status\" in ''|*[!0-9]*) status=null ;; esac", "case \"$attempt\" in ''|*[!0-9]*) attempt=0 ;; esac", + // A skip is the runner's own word, written before any container ran; + // a container that exits non-zero, 75 included, is a failure. "if [ \"$result\" = timeout ]; then outcome=timeout", - "elif [ \"$status\" = 75 ]; then outcome=skipped", + "elif [ -n \"$skipped\" ]; then outcome=skipped", "elif [ \"$result\" = success ] && [ \"$status\" = 0 ]; then outcome=success", "else outcome=failure; fi", "finished_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", @@ -458,8 +483,8 @@ func scheduleRunRecordLines(application, unit, job, state string) []string { "duration=0", "case \"$started_epoch\" in ''|*[!0-9]*) ;; *) duration=$((now - started_epoch)) ;; esac", "[ -z \"$started_at\" ] && started_at=$finished_at", - "record=$(printf '{\"run\":\"%s\",\"job\":\"%s\",\"trigger\":\"%s\",\"operation\":\"%s\",\"release\":\"%s\",\"started_at\":\"%s\",\"finished_at\":\"%s\",\"duration_s\":%s,\"attempts\":%s,\"exit_status\":%s,\"outcome\":\"%s\",\"inputs\":{%s}}' " + - "\"${INVOCATION_ID:-}\" " + q(job) + " \"$trigger\" \"$operation\" \"$release\" \"$started_at\" \"$finished_at\" \"$duration\" \"$attempt\" \"$status\" \"$outcome\" \"$inputs\")", + "record=$(printf '{\"run\":\"%s\",\"job\":\"%s\",\"trigger\":\"%s\",\"operation\":\"%s\",\"release\":\"%s\",\"started_at\":\"%s\",\"finished_at\":\"%s\",\"duration_s\":%s,\"attempts\":%s,\"exit_status\":%s,\"outcome\":\"%s\",\"reason\":\"%s\",\"inputs\":{%s}}' " + + "\"${INVOCATION_ID:-}\" " + q(job) + " \"$trigger\" \"$operation\" \"$release\" \"$started_at\" \"$finished_at\" \"$duration\" \"$attempt\" \"$status\" \"$outcome\" \"$skipped\" \"$inputs\")", "printf 'MESSAGE=%s\\nPRIORITY=6\\nSYSLOG_IDENTIFIER=" + scheduleRunIdentifier + "\\nONEBOX_APP=%s\\nONEBOX_UNIT=%s\\nONEBOX_JOB=%s\\n' " + "\"$record\" " + q(application) + " " + q(unit) + " " + q(job) + " | logger --journald || true", } diff --git a/internal/engine/schedule_history.go b/internal/engine/schedule_history.go index 71c8c93..b154125 100644 --- a/internal/engine/schedule_history.go +++ b/internal/engine/schedule_history.go @@ -16,18 +16,20 @@ import ( // scheduled run ends. The journal is the store: there is no file to trim and // nothing that can disagree with the unit's own log. type ScheduleRunRecord struct { - Run string `json:"run"` - Job string `json:"job"` - Trigger string `json:"trigger"` - Operation string `json:"operation,omitempty"` - Release string `json:"release,omitempty"` - StartedAt string `json:"started_at"` - FinishedAt string `json:"finished_at"` - DurationSeconds int `json:"duration_s"` - Attempts int `json:"attempts"` - ExitStatus *int `json:"exit_status"` - Outcome string `json:"outcome"` - Inputs map[string]string `json:"inputs,omitempty"` + Run string `json:"run"` + Job string `json:"job"` + Trigger string `json:"trigger"` + Operation string `json:"operation,omitempty"` + Release string `json:"release,omitempty"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + DurationSeconds int `json:"duration_s"` + Attempts int `json:"attempts"` + ExitStatus *int `json:"exit_status"` + Outcome string `json:"outcome"` + // Reason is set on a skipped run: what the runner met instead of running. + Reason string `json:"reason,omitempty"` + Inputs map[string]string `json:"inputs,omitempty"` } // ScheduleListing is one declared job beside its timer as the host reports it. @@ -150,25 +152,26 @@ func (e *Engine) ScheduleList(ctx context.Context) ([]ScheduleListing, error) { return out, nil } -// ScheduleLogs streams the journal of one run. The run id is systemd's -// invocation id, so the output is exactly that activation and nothing else. -// With no run given, the newest record's run is used. -func (e *Engine) ScheduleLogs(ctx context.Context, name, run string, stdout, stderr io.Writer) error { +// ScheduleLogs streams the journal of one run and returns the run id it +// streamed. The run id is systemd's invocation id, so the output is exactly +// that activation and nothing else. With no run given, the newest record's run +// is used, and the caller learns which one that was. +func (e *Engine) ScheduleLogs(ctx context.Context, name, run string, stdout, stderr io.Writer) (string, error) { if _, err := e.scheduledJob(name); err != nil { - return err + return "", err } if run == "" { records, err := e.ScheduleHistory(ctx, name, 1) if err != nil { - return err + return "", err } if len(records) == 0 { - return fmt.Errorf("job %s has no recorded runs", name) + return "", fmt.Errorf("job %s has no recorded runs", name) } run = records[0].Run } if !scheduleRunID.MatchString(run) { - return fmt.Errorf("run id %q is not a systemd invocation id", run) + return "", fmt.Errorf("run id %q is not a systemd invocation id", run) } - return e.T.RunStream(ctx, "journalctl _SYSTEMD_INVOCATION_ID="+run+" --no-pager -o short-iso", stdout, stderr) + return run, e.T.RunStream(ctx, "journalctl _SYSTEMD_INVOCATION_ID="+run+" --no-pager -o short-iso", stdout, stderr) } diff --git a/internal/engine/schedule_history_test.go b/internal/engine/schedule_history_test.go index 665bfe7..8418209 100644 --- a/internal/engine/schedule_history_test.go +++ b/internal/engine/schedule_history_test.go @@ -93,14 +93,18 @@ func TestScheduleLogsTargetsOneInvocation(t *testing.T) { return transport.Result{}, false } var out bytes.Buffer - if err := e.ScheduleLogs(context.Background(), "nightly", "", &out, &out); err != nil { + run, err := e.ScheduleLogs(context.Background(), "nightly", "", &out, &out) + if err != nil { t.Fatal(err) } + if run != "b2c3d4e5f60718293a4b5c6d7e8f9012" { + t.Fatalf("resolved run id = %q; the caller must learn which run was streamed", run) + } seq := strings.Join(f.Commands, "\n") if !strings.Contains(seq, "journalctl _SYSTEMD_INVOCATION_ID=b2c3d4e5f60718293a4b5c6d7e8f9012") { t.Fatalf("logs did not target the newest run:\n%s", seq) } - if err := e.ScheduleLogs(context.Background(), "nightly", "../etc", &out, &out); err == nil { + if _, err := e.ScheduleLogs(context.Background(), "nightly", "../etc", &out, &out); err == nil { t.Fatal("an invalid run id reached the shell") } } @@ -108,7 +112,7 @@ func TestScheduleLogsTargetsOneInvocation(t *testing.T) { func TestScheduleLogsRefusesAJobWithoutRecords(t *testing.T) { e, f := scheduledFixture(t) var out bytes.Buffer - err := e.ScheduleLogs(context.Background(), "nightly", "", &out, &out) + _, err := e.ScheduleLogs(context.Background(), "nightly", "", &out, &out) if err == nil || !strings.Contains(err.Error(), "no recorded runs") { t.Fatalf("err = %v", err) } diff --git a/internal/engine/schedule_run.go b/internal/engine/schedule_run.go index 406bdbf..c4c6696 100644 --- a/internal/engine/schedule_run.go +++ b/internal/engine/schedule_run.go @@ -6,6 +6,7 @@ import ( "fmt" "sort" "strings" + "time" "github.com/labstack/onebox/internal/app" "github.com/labstack/onebox/internal/journal" @@ -90,6 +91,14 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu if res.ExitCode != 0 { return result, fmt.Errorf("a manual run of %s is already pending (%s exists); wait for it, or remove the file on the host", name, path) } + // From here on the file is ours to clean up: a request that fails before + // the unit starts must not leave it behind to refuse the next one. + pending := true + defer func() { + if pending { + e.discardInputs(ctx, path) + } + }() writer := &journal.Writer{ T: e.T, Names: e.names(), DeployID: operationID, Epoch: epoch, Operator: journal.DefaultOperator(), @@ -121,23 +130,25 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu if err != nil { return result, err } + if res.ExitCode != 0 && !wait { + return result, fmt.Errorf("systemctl start %s: %s", unit, strings.TrimSpace(res.Stderr)) + } + // The unit was activated, so the runner owns the file now, whether it ran + // or skipped; a blocking start that exits non-zero still activated it. + pending = false result.Started = true if !wait { - if res.ExitCode != 0 { - return result, fmt.Errorf("systemctl start %s: %s", unit, strings.TrimSpace(res.Stderr)) - } e.logf("schedule: %s started as %s; ob schedule history %s shows the outcome", name, operationID, name) return result, nil } - records, err := e.ScheduleHistory(ctx, name, 1) + last, err := e.awaitScheduleRecord(ctx, name, operationID) if err != nil { + if res.ExitCode != 0 { + return result, fmt.Errorf("%w; systemctl start exited %d: %s", err, res.ExitCode, strings.TrimSpace(res.Stderr)) + } return result, err } - if len(records) == 0 { - return result, fmt.Errorf("job %s ran (systemctl exit %d) but left no run record; the host's notifier did not write one", name, res.ExitCode) - } - last := records[0] - result.Record = &last + result.Record = last exit := "-" if last.ExitStatus != nil { exit = fmt.Sprint(*last.ExitStatus) @@ -145,7 +156,7 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu e.logf("schedule: %s run %s %s after %d attempt(s) in %ds (exit %s)", name, last.Run, last.Outcome, last.Attempts, last.DurationSeconds, exit) // The operator asked for this run and waited for it, so anything but a - // success is a failure of the request, a skip included: the unit exits 75 + // success is a failure of the request, a skip included: the unit exits // cleanly, but the work was not done. if last.Outcome != "success" { return result, fmt.Errorf("job %s run %s ended %s; see ob schedule logs %s --run %s", name, last.Run, last.Outcome, name, last.Run) @@ -153,6 +164,38 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu return result, nil } +// awaitScheduleRecord finds the record of this operation's run. The notifier +// writes it from ExecStopPost and journald ingests it a moment after the +// blocking start returns, so a few short retries stand between the start and +// the read. Matching on the operation id means a record left by an earlier +// run, or by a timer firing that took this slot, is never reported as ours. +func (e *Engine) awaitScheduleRecord(ctx context.Context, name, operationID string) (*ScheduleRunRecord, error) { + for attempt := 0; attempt < 10; attempt++ { + if attempt > 0 { + e.Opts.Sleep(200 * time.Millisecond) + } + records, err := e.ScheduleHistory(ctx, name, 5) + if err != nil { + return nil, err + } + for i := range records { + if records[i].Operation == operationID { + return &records[i], nil + } + } + } + return nil, fmt.Errorf("no run record carries operation %s for job %s: the unit did not run for this request; a timer firing may have taken the slot, or the host's notifier wrote nothing", operationID, name) +} + +// discardInputs removes a pending inputs file this request wrote and can no +// longer hand to a run. Best effort: the file is root-only state on the host, +// and the error the caller is already returning is the one that matters. +func (e *Engine) discardInputs(ctx context.Context, path string) { + if _, err := e.T.Run(ctx, "rm -f "+q(path)); err != nil { + e.warnf("could not remove the pending inputs file %s: %v", path, err) + } +} + // scheduleInputsFile is the one-shot file the runner consumes: the operation // id on its reserved line, then one declared override per line. Values were // validated against a charset that has no newline or quote, so the format diff --git a/internal/engine/schedule_run_test.go b/internal/engine/schedule_run_test.go index 572826d..2feade0 100644 --- a/internal/engine/schedule_run_test.go +++ b/internal/engine/schedule_run_test.go @@ -132,14 +132,17 @@ func TestScheduleRunWaitReportsTheRecordAndFailsOnAnyOtherOutcome(t *testing.T) case strings.Contains(cmd, "systemctl start 'ob-sample-sync.service'"): return transport.Result{}, true case strings.Contains(cmd, "SYSLOG_IDENTIFIER=ob-run"): - return transport.Result{Stdout: `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"sync","trigger":"manual","started_at":"2026-09-05T15:00:01Z","finished_at":"2026-09-05T15:00:02Z","duration_s":1,"attempts":1,"exit_status":0,"outcome":"` + outcome + `","inputs":{}}` + "\n"}, true + // Newest first: a stale record from an earlier run precedes ours, + // and must not be mistaken for it. + return transport.Result{Stdout: `{"run":"ffffffffffffffffffffffffffffffff","job":"sync","trigger":"timer","operation":"","started_at":"2026-09-05T14:00:01Z","finished_at":"2026-09-05T14:00:02Z","duration_s":1,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}}` + "\n" + + `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"sync","trigger":"manual","operation":"op-` + outcome + `","started_at":"2026-09-05T15:00:01Z","finished_at":"2026-09-05T15:00:02Z","duration_s":1,"attempts":1,"exit_status":0,"outcome":"` + outcome + `","inputs":{}}` + "\n"}, true } return base(cmd) } e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) result, err := e.ScheduleRun(context.Background(), "op-"+outcome, "sync", nil, true) - if result.Record == nil || result.Record.Outcome != outcome { - t.Fatalf("%s: record not returned: %#v", outcome, result) + if result.Record == nil || result.Record.Outcome != outcome || result.Record.Run != "a1b2c3d4e5f60718293a4b5c6d7e8f90" { + t.Fatalf("%s: this operation's record not returned: %#v", outcome, result) } if (err != nil) != wantErr { t.Fatalf("%s: err = %v, wantErr %v", outcome, err, wantErr) @@ -149,3 +152,34 @@ func TestScheduleRunWaitReportsTheRecordAndFailsOnAnyOtherOutcome(t *testing.T) } } } + +func TestScheduleRunDiscardsItsInputsWhenTheStartFails(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog", "prices"}, Default: "catalog"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "systemctl start"): + return transport.Result{ExitCode: 5, Stderr: "Unit ob-sample-sync.service not found."}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + _, err := e.ScheduleRun(context.Background(), "op-1", "sync", map[string]string{"SOURCE": "prices"}, false) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("start failure was not reported: %v", err) + } + seq := strings.Join(f.Commands, "\n") + if !strings.Contains(seq, "rm -f '/var/lib/ob/sample/schedule/sync.inputs'") { + t.Fatalf("a failed start left the inputs file pending:\n%s", seq) + } +} diff --git a/internal/engine/schedule_status.go b/internal/engine/schedule_status.go index 8fa0c4d..514a545 100644 --- a/internal/engine/schedule_status.go +++ b/internal/engine/schedule_status.go @@ -10,41 +10,44 @@ import ( "github.com/labstack/onebox/internal/release" ) -// StatusSchedule is the host-observed state of one declared scheduled job. -// systemd's Result and exit status are reported as observed; the verdict comes -// from the run records the notifier writes to the journal: outcome, attempts, -// duration, and how many firings in a row have failed. +// StatusSchedule is the host-observed state of one declared scheduled job. The +// verdict comes from the run records the notifier writes to the journal: +// outcome, attempts, duration, and how many firings in a row have failed or +// been skipped. systemd contributes the timer's state and next elapse and +// whether a run is in progress. type StatusSchedule struct { - Name string `json:"name"` - Unit string `json:"unit"` - TimerState string `json:"timer_state"` - Running bool `json:"running"` - DeployLock string `json:"deploy_lock"` - Timeout string `json:"timeout"` - PinnedRelease string `json:"pinned_release,omitempty"` - StartedAt string `json:"started_at,omitempty"` - LastResult string `json:"last_result"` - LastExitStatus int `json:"last_exit_status,omitempty"` - Diverged bool `json:"diverged"` - Issues []string `json:"issues,omitempty"` + Name string `json:"name"` + Unit string `json:"unit"` + TimerState string `json:"timer_state"` + Running bool `json:"running"` + DeployLock string `json:"deploy_lock"` + Timeout string `json:"timeout"` + PinnedRelease string `json:"pinned_release,omitempty"` + StartedAt string `json:"started_at,omitempty"` + Diverged bool `json:"diverged"` + Issues []string `json:"issues,omitempty"` - // From the timer and the run records. NextRun string `json:"next_run,omitempty"` Attempt int `json:"attempt,omitempty"` LastOutcome string `json:"last_outcome,omitempty"` + LastReason string `json:"last_reason,omitempty"` LastDurationSeconds int `json:"last_duration_s,omitempty"` LastAttempts int `json:"last_attempts,omitempty"` ConsecutiveFailures int `json:"consecutive_failures,omitempty"` + ConsecutiveSkips int `json:"consecutive_skips,omitempty"` // JournalPersistent is false when the host keeps its journal in memory, so // the records above only reach back to the last boot. JournalPersistent bool `json:"journal_persistent"` } +// skipStreakIssue is how many firings in a row may be skipped before status +// says so. One skip is timing; a streak is a job that never runs, which is the +// failure mode a skipped record exists to expose. +const skipStreakIssue = 3 + type scheduleUnitObservation struct { loadState string activeState string - result string - exitStatus int release string startedAt string attempt string @@ -69,7 +72,7 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) unit := e.names().ScheduledJobUnit(job.Name) commands = append(commands, "printf '%s\\n' "+q("@@"+job.Name+":service"), - "systemctl show "+q(unit+".service")+" --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus", + "systemctl show "+q(unit+".service")+" --no-pager --property=LoadState --property=ActiveState", "printf '%s\\n' "+q("@@"+job.Name+":timer"), "systemctl show "+q(unit+".timer")+" --no-pager --property=LoadState --property=ActiveState --property=NextElapseUSecRealtime", "printf '%s\\n' "+q("@@"+job.Name+":run"), @@ -95,13 +98,11 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) if name == "" || kind == "" { return } - exit, _ := strconv.Atoi(values["ExecMainStatus"]) if observed[name] == nil { observed[name] = map[string]scheduleUnitObservation{} } observed[name][kind] = scheduleUnitObservation{ loadState: values["LoadState"], activeState: values["ActiveState"], - result: values["Result"], exitStatus: exit, release: values["release"], startedAt: values["started_at"], attempt: values["attempt"], next: values["NextElapseUSecRealtime"], history: parseScheduleRunRecords(strings.Join(raw, "\n")), @@ -147,7 +148,6 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) status := StatusSchedule{ Name: job.Name, Unit: unit, TimerState: timer.activeState, Running: service.activeState == "activating", DeployLock: job.DeployLock, Timeout: job.Timeout, - LastResult: service.result, LastExitStatus: service.exitStatus, NextRun: timer.next, JournalPersistent: journalPersistent, } if status.Running { @@ -165,10 +165,18 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) if len(records) > 0 { last := records[0] status.LastOutcome = last.Outcome + status.LastReason = last.Reason status.LastDurationSeconds = last.DurationSeconds status.LastAttempts = last.Attempts - // A skip says nothing about the job, so it neither breaks nor - // extends a failure streak. + // Skips say nothing about the job, so they neither break nor + // extend a failure streak; their own streak is counted from the + // newest record until something actually ran. + for _, record := range records { + if record.Outcome != "skipped" { + break + } + status.ConsecutiveSkips++ + } for _, record := range records { if record.Outcome == "skipped" { continue @@ -185,9 +193,8 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) if service.loadState != "loaded" { status.Issues = append(status.Issues, "service unit is not loaded") } - // The record is the verdict. systemd's Result and ExecMainStatus are - // still reported as observed, but only a recorded failure or timeout - // is an issue. + // The record is the verdict: a recorded failure or timeout is an + // issue, and so is a job that keeps being skipped. if status.LastOutcome == "failure" || status.LastOutcome == "timeout" { exit := "?" if records[0].ExitStatus != nil { @@ -195,6 +202,9 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) } status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %s)", status.LastOutcome, exit)) } + if status.ConsecutiveSkips >= skipStreakIssue { + status.Issues = append(status.Issues, fmt.Sprintf("skipped %d firings in a row: %s", status.ConsecutiveSkips, status.LastReason)) + } status.Diverged = len(status.Issues) > 0 statuses = append(statuses, status) } diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index 503cbc5..2ccfc50 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -179,7 +179,7 @@ func TestScheduledJobUnitContract(t *testing.T) { Calendar: "*-*-* 02:00:00", Timeout: "45m", CatchUp: false, DeployLock: "exclusive", } names := app.Names{App: "sample", BasePath: "/var/lib/ob"} - runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil) + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) service := scheduleServiceUnit("sample", job, "/etc/systemd/system/ob-sample-nightly.run", "/etc/systemd/system/ob-sample-nightly.notify") @@ -187,9 +187,9 @@ func TestScheduledJobUnitContract(t *testing.T) { for _, want := range []string{ "exec 9>'/var/lib/ob/sample/schedule/nightly.lock'", - "flock --exclusive --nonblock --conflict-exit-code 75 9", + "flock --exclusive --nonblock 9 || skip", "exec 8>'/var/lib/ob/sample/schedule.lock'", - "flock --exclusive --nonblock --conflict-exit-code 75 8", + "flock --exclusive --nonblock 8 || skip", "/var/lib/ob/sample/lock", "application operation holds the deploy lock", "docker compose", @@ -241,11 +241,11 @@ func TestPinnedScheduledJobRunnerLeasesImmutableRelease(t *testing.T) { runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", []app.EnvFile{ {File: "config/runtime.env"}, {File: "secrets/runtime.env", Provider: "sops"}, - }) + }, 10*time.Minute) for _, want := range []string{ "exec 9>'/var/lib/ob/sample/schedule/refresh.lock'", - "flock --exclusive --nonblock --conflict-exit-code 75 9", + "flock --exclusive --nonblock 9 || skip", "exec 8>'/var/lib/ob/sample/schedule.lock'", "release_dir=$(readlink -f '/var/lib/ob/sample/current')", "exec 7>>\"$release_dir/.ob-schedule.lease\"", @@ -341,7 +341,7 @@ func TestPinnedScheduledJobLockProtocol(t *testing.T) { } job := app.ScheduledJob{Name: "refresh", DeployLock: "pinned"} - runner := scheduleRunnerScript("sample", job, names, filepath.Join(names.AppDir(), "lock"), nil) + runner := scheduleRunnerScript("sample", job, names, filepath.Join(names.AppDir(), "lock"), nil, 10*time.Minute) runner = strings.ReplaceAll(runner, "/usr/bin/docker", q(stub)) command := exec.CommandContext(ctx, "sh") command.Stdin = strings.NewReader(runner) @@ -427,8 +427,10 @@ func TestPinnedScheduledJobLockProtocol(t *testing.T) { if err != nil || len(leases) != 0 { t.Fatalf("completed release remained leased: leases=%v err=%v", leases, err) } - if _, err := os.Stat(names.ScheduledJobRunState(job.Name)); !os.IsNotExist(err) { - t.Fatalf("runner state survived completion: %v", err) + // The notifier, not the runner, removes the state: it is the evidence + // ExecStopPost turns into the run record. + if _, err := os.Stat(names.ScheduledJobRunState(job.Name)); err != nil { + t.Fatalf("runner removed the state the notifier finalises: %v", err) } } @@ -504,41 +506,6 @@ func TestSyncSchedulesRefusesMissingFlockBeforeInstallingUnits(t *testing.T) { } } -func TestScheduleStatusSurfacesTheLastSystemdFailure(t *testing.T) { - cfg := testConfig() - cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", - Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, - } - f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { - if strings.Contains(cmd, "systemctl show") { - return transport.Result{Stdout: `@@nightly:service -LoadState=loaded -ActiveState=failed -Result=timeout -ExecMainStatus=15 -@@nightly:timer -LoadState=loaded -ActiveState=active -`}, true - } - return transport.Result{}, false - }} - e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) - statuses, err := e.scheduleStatuses(context.Background()) - if err != nil { - t.Fatal(err) - } - // systemd's own result is reported, but the verdict is the run record, - // and there is none here. - if len(statuses) != 1 || statuses[0].LastResult != "timeout" || statuses[0].LastExitStatus != 15 { - t.Fatalf("systemd result was not surfaced: %#v", statuses) - } - if statuses[0].Diverged || len(statuses[0].Issues) != 0 { - t.Fatalf("an issue was raised without a run record: %#v", statuses[0]) - } -} - func TestScheduleStatusReportsRunningPinnedRelease(t *testing.T) { cfg := testConfig() cfg.Workloads["refresh"] = app.Workload{ @@ -836,7 +803,7 @@ func TestScheduledJobRunnersRecordRunStateForTheNotifier(t *testing.T) { {"pinned", app.ScheduledJob{Name: "nightly", Cron: "0 2 * * *", Timezone: "UTC", Calendar: "*-*-* 02:00:00", Timeout: "45m", DeployLock: "pinned"}}, } { t.Run(tc.name, func(t *testing.T) { - runner := scheduleRunnerScript("sample", tc.job, names, "/var/lib/ob/sample/lock", nil) + runner := scheduleRunnerScript("sample", tc.job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) for _, want := range []string{ "state='/var/lib/ob/sample/schedule/nightly.state'", "write_state() {", @@ -861,8 +828,8 @@ func TestScheduledJobRunnersRecordRunStateForTheNotifier(t *testing.T) { } service := scheduleServiceUnit("sample", app.ScheduledJob{Name: "nightly", Timeout: "45m"}, "/etc/systemd/system/ob-sample-nightly.run", "/etc/systemd/system/ob-sample-nightly.notify") - if !strings.Contains(service, "SuccessExitStatus=75") { - t.Errorf("a lock-conflict skip must not be a failed unit:\n%s", service) + if strings.Contains(service, "SuccessExitStatus") { + t.Errorf("a skip is recorded by the runner and exits 0; the unit needs no exit-status remap:\n%s", service) } } @@ -885,7 +852,7 @@ func TestScheduledJobNotifierWritesOneRunRecordToTheJournal(t *testing.T) { `outcome=success`, `outcome=failure`, `"run":"%s","job":"%s","trigger":"%s","operation":"%s","release":"%s"`, - `"duration_s":%s,"attempts":%s,"exit_status":%s,"outcome":"%s","inputs":{%s}`, + `"duration_s":%s,"attempts":%s,"exit_status":%s,"outcome":"%s","reason":"%s","inputs":{%s}`, `"${INVOCATION_ID:-}" 'nightly'`, `SYSLOG_IDENTIFIER=ob-run\nONEBOX_APP=%s\nONEBOX_UNIT=%s\nONEBOX_JOB=%s`, `"$record" 'sample' 'ob-sample-nightly' 'nightly' | logger --journald`, @@ -987,11 +954,12 @@ func TestScheduledJobNotifierRecordsEachOutcomeAndRemovesState(t *testing.T) { exit any attempt float64 }{ - "success": {state, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "a1b2"}, "success", float64(0), 2}, - "failure": {state, map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "1"}, "failure", float64(1), 2}, - "timeout": {state, map[string]string{"SERVICE_RESULT": "timeout", "EXIT_STATUS": "TERM"}, "timeout", nil, 2}, - "skipped": {"", map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "75", "TRIGGER_UNIT": "ob-sample-nightly.timer"}, "skipped", float64(75), 0}, - "no state": {"", map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "3"}, "failure", float64(3), 0}, + "success": {state, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "a1b2"}, "success", float64(0), 2}, + "failure": {state, map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "1"}, "failure", float64(1), 2}, + "timeout": {state, map[string]string{"SERVICE_RESULT": "timeout", "EXIT_STATUS": "TERM"}, "timeout", nil, 2}, + "skipped": {"skipped=another run of this job is still in progress\noperation=\ninputs=\n", map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "TRIGGER_UNIT": "ob-sample-nightly.timer"}, "skipped", float64(0), 0}, + "job exits 75": {state, map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "75"}, "failure", float64(75), 2}, + "no state": {"", map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "3"}, "failure", float64(3), 0}, } { t.Run(name, func(t *testing.T) { record, stateLeft, sends := runNotifier(t, app.ScheduledJob{Name: "nightly", Notify: []string{"failure", "timeout"}}, nil, tc.state, tc.env) @@ -1004,11 +972,11 @@ func TestScheduledJobNotifierRecordsEachOutcomeAndRemovesState(t *testing.T) { if record["job"] != "nightly" || record["run"] != tc.env["INVOCATION_ID"] { t.Fatalf("identity fields wrong: %#v", record) } - if tc.state != "" && (record["release"] != "20260905-140000-ab12cd" || record["trigger"] != "timer" || record["duration_s"].(float64) < 1) { + if tc.state == state && (record["release"] != "20260905-140000-ab12cd" || record["trigger"] != "timer" || record["duration_s"].(float64) < 1) { t.Fatalf("state fields not carried: %#v", record) } - if name == "skipped" && record["trigger"] != "timer" { - t.Fatalf("trigger not derived from TRIGGER_UNIT: %#v", record) + if name == "skipped" && (record["trigger"] != "timer" || record["reason"] != "another run of this job is still in progress") { + t.Fatalf("skip was not recorded with its trigger and reason: %#v", record) } if stateLeft { t.Fatal("state file survived the notifier") @@ -1109,7 +1077,7 @@ func TestScheduledJobRunnerRetriesWithCappedDoublingBackoff(t *testing.T) { job := app.ScheduledJob{Name: "nightly", Timeout: "45m", DeployLock: "exclusive", RetryAttempts: 3, RetryBackoff: 30 * time.Second, RetryMaxBackoff: 10 * time.Minute} names := app.Names{App: "sample", BasePath: "/var/lib/ob"} - runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil) + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) for _, want := range []string{ "max_attempts=3", "backoff=30", "max_backoff=600", "attempt=1", "while :; do", "write_state \"$attempt\"", @@ -1121,11 +1089,11 @@ func TestScheduledJobRunnerRetriesWithCappedDoublingBackoff(t *testing.T) { t.Errorf("runner is missing %q:\n%s", want, runner) } } - single := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil) + single := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) if strings.Contains(single, "while :; do") { t.Errorf("a single-attempt job must not carry a retry loop:\n%s", single) } - pinned := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "pinned", RetryAttempts: 2, RetryBackoff: time.Second, RetryMaxBackoff: time.Minute}, names, "/var/lib/ob/sample/lock", nil) + pinned := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "pinned", RetryAttempts: 2, RetryBackoff: time.Second, RetryMaxBackoff: time.Minute}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) if !strings.Contains(pinned, "while :; do") || strings.Index(pinned, "flock --unlock 8") > strings.Index(pinned, "while :; do") { t.Errorf("pinned runner must release the schedule mutex before its attempt loop:\n%s", pinned) } @@ -1182,11 +1150,15 @@ func TestScheduledJobNotifierSendsOnlySelectedOutcomesWithTheRunID(t *testing.T) "failure selected": {[]string{"failure", "timeout"}, map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "1", "INVOCATION_ID": "abc123"}, 2, "fail"}, "success not selected": {[]string{"failure", "timeout"}, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "abc123"}, 0, ""}, "success selected": {[]string{"success"}, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "abc123"}, 1, "ok"}, - "skipped selected": {[]string{"skipped"}, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "75", "INVOCATION_ID": "abc123"}, 2, "fail"}, + "skipped selected": {[]string{"skipped"}, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "abc123"}, 2, "fail"}, "timeout not selected": {[]string{"failure"}, map[string]string{"SERVICE_RESULT": "timeout", "EXIT_STATUS": "TERM", "INVOCATION_ID": "abc123"}, 0, ""}, } { t.Run(name, func(t *testing.T) { - _, _, sends := runNotifier(t, app.ScheduledJob{Name: "nightly", Notify: tc.notify}, webhooks, state, tc.env) + runState := state + if name == "skipped selected" { + runState = "skipped=an application operation holds the deploy lock\noperation=\ninputs=\n" + } + _, _, sends := runNotifier(t, app.ScheduledJob{Name: "nightly", Notify: tc.notify}, webhooks, runState, tc.env) calls := 0 for _, arg := range sends { if arg == "--" { @@ -1230,7 +1202,7 @@ func TestScheduledJobRunnerConsumesManualInputsWithoutShellInterpolation(t *test job := app.ScheduledJob{Name: "sync", Timeout: "45m", DeployLock: "pinned", RetryAttempts: 1, Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog"}, Default: "catalog"}}} names := app.Names{App: "sample", BasePath: "/var/lib/ob"} - runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil) + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) for _, want := range []string{ "inputs_file='/var/lib/ob/sample/schedule/sync.inputs'", `if [ -z "${TRIGGER_UNIT:-}" ] && [ -f "$inputs_file" ]; then`, @@ -1254,16 +1226,16 @@ func TestScheduledJobRunnerConsumesManualInputsWithoutShellInterpolation(t *test } // The consume block precedes the locks so a skipped manual run cannot // leave its inputs for the next timer firing. - if strings.Index(runner, "inputs_file=") > strings.Index(runner, "flock --exclusive --nonblock --conflict-exit-code 75 9") { + if strings.Index(runner, "inputs_file=") > strings.Index(runner, "flock --exclusive --nonblock 9") { t.Fatalf("inputs are consumed after the lock:\n%s", runner) } - exclusive := scheduleRunnerScript("sample", app.ScheduledJob{Name: "sync", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil) + exclusive := scheduleRunnerScript("sample", app.ScheduledJob{Name: "sync", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) if !strings.Contains(exclusive, "inputs_file=") || !strings.Contains(exclusive, `run --rm --no-deps "$@" --name`) { t.Fatalf("exclusive runner does not consume inputs:\n%s", exclusive) } } -func TestSyncSchedulesRequiresSystemd252ForInputs(t *testing.T) { +func TestSyncSchedulesRequireSystemd252ForEveryScheduledJob(t *testing.T) { cfg := testConfig() cfg.Workloads["sync"] = app.Workload{ Role: app.RoleJob, When: "manual", DataEffect: "none", @@ -1289,7 +1261,7 @@ func TestSyncSchedulesRequiresSystemd252ForInputs(t *testing.T) { e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) err := e.SyncSchedules(context.Background()) if wantErr && (err == nil || !strings.Contains(err.Error(), "systemd 252")) { - t.Fatalf("version %q was accepted for a job with inputs: %v", version, err) + t.Fatalf("version %q was accepted for a scheduled job: %v", version, err) } if !wantErr && err != nil { t.Fatalf("version %q was refused: %v", version, err) @@ -1355,3 +1327,45 @@ func TestScheduleInputsLinesParseTheFileIntoArguments(t *testing.T) { } } } + +func TestScheduleStatusRaisesAnIssueForASkipStreak(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + skip := `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:01Z","duration_s":0,"attempts":0,"exit_status":0,"outcome":"skipped","reason":"an application operation holds the deploy lock","inputs":{}}` + success := `{"run":"c3d4e5f60718293a4b5c6d7e8f901234","job":"nightly","trigger":"timer","started_at":"2026-09-03T02:00:01Z","finished_at":"2026-09-03T02:01:02Z","duration_s":61,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}}` + for name, tc := range map[string]struct { + history string + skips int + issue bool + }{ + "two skips": {skip + "\n" + skip + "\n" + success, 2, false}, + "three skips": {skip + "\n" + skip + "\n" + skip + "\n" + success, 3, true}, + } { + t.Run(name, func(t *testing.T) { + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: "@@journal\npersistent\n@@nightly:service\nLoadState=loaded\nActiveState=inactive\n@@nightly:timer\nLoadState=loaded\nActiveState=active\n@@nightly:run\n@@nightly:history\n" + tc.history + "\n"}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + got := statuses[0] + if got.ConsecutiveSkips != tc.skips || got.ConsecutiveFailures != 0 || got.LastOutcome != "skipped" || got.LastReason == "" { + t.Fatalf("skips not counted: %#v", got) + } + if got.Diverged != tc.issue { + t.Fatalf("issue = %v, want %v: %#v", got.Diverged, tc.issue, got.Issues) + } + if tc.issue && !strings.Contains(strings.Join(got.Issues, "; "), "skipped 3 firings in a row: an application operation holds the deploy lock") { + t.Fatalf("issue does not name the streak and reason: %#v", got.Issues) + } + }) + } +} diff --git a/internal/engine/status.go b/internal/engine/status.go index 415c003..386f9be 100644 --- a/internal/engine/status.go +++ b/internal/engine/status.go @@ -171,10 +171,7 @@ func (e *Engine) Status(ctx context.Context) error { e.ui.Println(fmt.Sprintf("schedule %-11s %s", schedule.Name, e.ui.Warn(strings.Join(schedule.Issues, "; ")+" ⚠"))) continue } - result := schedule.LastResult - if result == "" { - result = "not run yet" - } + result := "not recorded" if schedule.Running { detail := fmt.Sprintf("running; policy: %s; timeout: %s", schedule.DeployLock, schedule.Timeout) if schedule.Attempt > 0 { @@ -190,7 +187,10 @@ func (e *Engine) Status(ctx context.Context) error { if schedule.NextRun != "" { detail += "; next: " + schedule.NextRun } - if schedule.LastOutcome != "" { + switch { + case schedule.LastOutcome == "skipped": + result = "skipped (" + schedule.LastReason + ")" + case schedule.LastOutcome != "": result = fmt.Sprintf("%s (%ds, %d attempt(s))", schedule.LastOutcome, schedule.LastDurationSeconds, schedule.LastAttempts) } detail += "; last: " + result diff --git a/internal/engine/status_snapshot_test.go b/internal/engine/status_snapshot_test.go index 0d053a3..f8e01e6 100644 --- a/internal/engine/status_snapshot_test.go +++ b/internal/engine/status_snapshot_test.go @@ -263,7 +263,7 @@ ActiveState=active if !snapshot.Complete || !snapshot.Diverged || len(snapshot.Schedules) != 1 { t.Fatalf("scheduled failure was not included as observed divergence: %#v", snapshot) } - if got := snapshot.Schedules[0]; !got.Diverged || got.LastOutcome != "failure" || got.LastResult != "exit-code" || got.LastExitStatus != 9 { + if got := snapshot.Schedules[0]; !got.Diverged || got.LastOutcome != "failure" { t.Fatalf("unexpected scheduled-job status: %#v", got) } } diff --git a/site/public/onebox.run-v1.schema.json b/site/public/onebox.run-v1.schema.json index 4344bd9..76ba0e3 100644 --- a/site/public/onebox.run-v1.schema.json +++ b/site/public/onebox.run-v1.schema.json @@ -1878,6 +1878,11 @@ "required": [ "schedule" ] + }, + { + "required": [ + "inputs" + ] } ] } @@ -2234,6 +2239,18 @@ "inputs": { "additionalProperties": { "additionalProperties": false, + "oneOf": [ + { + "required": [ + "enum" + ] + }, + { + "required": [ + "pattern" + ] + } + ], "patternProperties": { "^x-": {} }, @@ -2264,9 +2281,15 @@ "type": "string" } }, + "required": [ + "default" + ], "type": "object" }, "description": "Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them.", + "propertyNames": { + "pattern": "^[A-Z][A-Z0-9_]*$" + }, "type": "object" }, "labels": { @@ -2578,6 +2601,12 @@ "default": "failure, timeout", "description": "Run outcomes that send the configured notifications: success, failure, timeout, skipped.", "items": { + "enum": [ + "success", + "failure", + "timeout", + "skipped" + ], "type": "string" }, "type": "array" @@ -2595,22 +2624,26 @@ "examples": [ 3 ], + "maximum": 10, + "minimum": 1, "type": "integer" }, "backoff": { "default": "30s", - "description": "Sleep before the second attempt; it doubles after each failure.", + "description": "Sleep before the second attempt; it doubles after each failure. Expects a duration such as 30s, 5m, 1h30m or 14d.", "examples": [ "1m" ], + "pattern": "^(([0-9]+([.][0-9]+)?(ns|us|µs|ms|s|m|h))+|[0-9]+d)$", "type": "string" }, "max_backoff": { "default": "10m", - "description": "Upper bound for the doubling sleep.", + "description": "Upper bound for the doubling sleep. Expects a duration such as 30s, 5m, 1h30m or 14d.", "examples": [ "30m" ], + "pattern": "^(([0-9]+([.][0-9]+)?(ns|us|µs|ms|s|m|h))+|[0-9]+d)$", "type": "string" } }, diff --git a/site/src/content/docs/guides/schedule-a-job.mdx b/site/src/content/docs/guides/schedule-a-job.mdx index 08359e7..5a8f439 100644 --- a/site/src/content/docs/guides/schedule-a-job.mdx +++ b/site/src/content/docs/guides/schedule-a-job.mdx @@ -77,7 +77,10 @@ release lease is held instead and every attempt runs the same release. A host-fired job takes an application-wide kernel lock for its whole container run. Every operation that establishes Onebox's fenced application lock takes the same scheduling mutex while publishing its ownership. A timer that collides -fails visibly instead of modifying Docker beside a deploy. +does not modify Docker beside a deploy: it records a `skipped` run with the +reason and exits cleanly. An application lock older than its TTL is treated as +expired, exactly as a deploy treats it, so a runner that died mid-operation +cannot silence a timer forever. The target must provide `flock` (part of `util-linux` on supported Linux hosts). Onebox refuses to install or run schedules when that serialization primitive is @@ -137,15 +140,17 @@ its `ONEBOX_UNIT` field, so `journalctl SYSLOG_IDENTIFIER=ob-run ONEBOX_UNIT=ob-- -o cat` on the host is the raw history: ```json -{"run":"a3f9…","job":"nightly-dump","trigger":"timer","operation":"","release":"20260905-140000-ab12cd","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:04:37Z","duration_s":276,"attempts":2,"exit_status":0,"outcome":"success","inputs":{}} +{"run":"a3f9…","job":"nightly-dump","trigger":"timer","operation":"","release":"20260905-140000-ab12cd","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:04:37Z","duration_s":276,"attempts":2,"exit_status":0,"outcome":"success","reason":"","inputs":{}} ``` `run` is systemd's invocation id, so the record and the run's own log share a key. `outcome` is one of `success`, `failure`, `timeout`, or `skipped`. A skip is a firing that met a running instance of the same job or an application -operation holding the deploy lock; it exits `75`, which the unit treats as a -clean exit, and it is recorded because a job that is silently never running -looks exactly like one that works. +operation holding the deploy lock. The runner writes the `reason` and exits +before any container starts, so the unit is not failed and the job's own exit +status is never mistaken for a skip. Skips are recorded because a job that is +silently never running looks exactly like one that works: one skip is timing, +and three in a row are reported by `ob status` as an issue. Read the records back from the workstation: @@ -267,11 +272,12 @@ operator-initiated run of it keeps the sealed plan, approval and backup-report gates of `ob job run` below. The runner tells a manual activation from a timer firing by the `TRIGGER_UNIT` -variable systemd sets on timer activations, which needs systemd 252 or newer -on the host. Ubuntu 24.04 and Debian 12 qualify; a job that declares inputs is -refused at deploy time on an older host. A job without inputs still runs on an -older host, but its records say `manual` for every activation, because nothing -there tells the runner otherwise. +variable systemd sets on timer activations, which that project introduced in +version 252. Every scheduled job therefore needs systemd 252 or newer on the +host, not only one with inputs: the record's `trigger` is part of the contract, +and on an older systemd a timer firing would both be recorded as manual and +consume an operator's pending inputs. `ob preflight` and `ob deploy` refuse an +older host before anything is staged. Ubuntu 24.04 and Debian 12 qualify. ## Running one by hand diff --git a/site/src/content/docs/reference/fields/workloads.mdx b/site/src/content/docs/reference/fields/workloads.mdx index 044559f..cfd038e 100644 --- a/site/src/content/docs/reference/fields/workloads.mdx +++ b/site/src/content/docs/reference/fields/workloads.mdx @@ -59,7 +59,7 @@ cannot drift from what `ob validate` accepts. | `.image.reference` | string | — | Complete container image reference, optionally tagged or digest-pinned. Expects a registry reference such as nginx:1.27 or ghcr.io/acme/app@sha256:…. | | `.init` | boolean | — | Run a minimal init process as PID 1 inside the container. | | `.inputs` | map | — | Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them. | -| `.inputs..default` | string | — | Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint. | +| `.inputs..default` `*` | string | — | Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint. | | `.inputs..description` | string | — | What the input controls. | | `.inputs..enum` | list | — | Accepted values. | | `.inputs..pattern` | string | — | Regular expression the whole value must match. | @@ -97,11 +97,11 @@ cannot drift from what `ob validate` accepts. | `.schedule.catch_up` | boolean | `true` | Run once after the host returns if an elapsed schedule was missed while it was offline. | | `.schedule.cron` | string | — | Five-field cron schedule translated to a host timer. Expects five cron fields. | | `.schedule.deploy_lock` | `exclusive` · `pinned` | `exclusive` | Deployment coordination policy: exclusive blocks application operations for the full run; pinned leases the immutable starting release and permits only deployments without data-changing jobs or untyped hooks. | -| `.schedule.notify` | list | `failure, timeout` | Run outcomes that send the configured notifications: success, failure, timeout, skipped. | +| `.schedule.notify` | list of `success` · `failure` · `timeout` · `skipped` | `failure, timeout` | Run outcomes that send the configured notifications: success, failure, timeout, skipped. | | `.schedule.retry` | object | — | Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run. | | `.schedule.retry.attempts` | integer | `1` | Total attempts including the first, 1 to 10. | -| `.schedule.retry.backoff` | string | `30s` | Sleep before the second attempt; it doubles after each failure. | -| `.schedule.retry.max_backoff` | string | `10m` | Upper bound for the doubling sleep. | +| `.schedule.retry.backoff` | string | `30s` | Sleep before the second attempt; it doubles after each failure. Expects a duration such as 30s, 5m, 1h30m or 14d. | +| `.schedule.retry.max_backoff` | string | `10m` | Upper bound for the doubling sleep. Expects a duration such as 30s, 5m, 1h30m or 14d. | | `.schedule.timeout` | string | `1h` | Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d. | | `.schedule.timezone` | string | `UTC` | IANA timezone used to interpret the cron schedule. Expects an IANA zone name such as UTC or Europe/Berlin. | | `.stdin_open` | boolean | — | Keep standard input open for the container. | From b4762c834a0d1c21043acadacb6cd285842cf5e8 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 21:15:20 -0700 Subject: [PATCH 18/25] fix(schedule): act on the Copilot review - The runner reads the application lock's age with the same shell AcquireLock uses, in whole seconds, so a sub-minute TTL is honoured as written and an unreadable lock still fails closed. - A journalctl failure is an error, not an empty history: unreadable and never-ran are different answers. - Writing the inputs file distinguishes an already-pending run from a host that refused the write. - Range over an integer, which the pinned linter asks for. Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule.go | 12 ++++--- internal/engine/schedule_history.go | 9 +++-- internal/engine/schedule_history_test.go | 14 ++++++++ internal/engine/schedule_run.go | 15 ++++++--- internal/engine/schedule_run_test.go | 43 ++++++++++++++++++++++++ internal/engine/schedule_test.go | 4 +-- 6 files changed, 84 insertions(+), 13 deletions(-) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 0aecac8..3e39a61 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -258,11 +258,13 @@ func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names // // The application lock is honoured for as long as AcquireLock would honour it: // a lock older than the TTL belongs to a runner that died, and AcquireLock -// takes it over, so the timer must not defer to it forever either. +// takes it over, so the timer must not defer to it forever either. The age +// comes from the same shell AcquireLock reads it with, in whole seconds, and +// that shell fails closed: an unreadable lock reads as fresh. func scheduleLockLines(names app.Names, job, applicationLock string, lockTTL time.Duration) []string { - ttlMinutes := int(math.Ceil(lockTTL.Minutes())) - if ttlMinutes < 1 { - ttlMinutes = 1 + ttlSeconds := int(math.Ceil(lockTTL.Seconds())) + if ttlSeconds < 1 { + ttlSeconds = 1 } return []string{ "state=" + q(names.ScheduledJobRunState(job)), @@ -274,7 +276,7 @@ func scheduleLockLines(names app.Names, job, applicationLock string, lockTTL tim "/usr/bin/flock --exclusive --nonblock 9 || skip 'another run of this job is still in progress'", "exec 8>" + q(names.ScheduleRunLock()), "/usr/bin/flock --exclusive --nonblock 8 || skip 'an application operation is taking its lock'", - "if [ -e " + q(applicationLock) + " ] && [ -z \"$(find " + q(applicationLock) + " -mmin +" + strconv.Itoa(ttlMinutes) + " 2>/dev/null)\" ]; then skip 'an application operation holds the deploy lock'; fi", + "if [ -e " + q(applicationLock) + " ] && [ \"$(" + lockAgeCmd(applicationLock) + ")\" -le " + strconv.Itoa(ttlSeconds) + " ]; then skip 'an application operation holds the deploy lock'; fi", } } diff --git a/internal/engine/schedule_history.go b/internal/engine/schedule_history.go index b154125..f6d86dc 100644 --- a/internal/engine/schedule_history.go +++ b/internal/engine/schedule_history.go @@ -50,13 +50,15 @@ type ScheduleListing struct { var scheduleRunID = regexp.MustCompile(`^[0-9a-f]{32}$`) // scheduleHistoryCommand matches the record's own fields rather than the -// unit journald attributed it to; see scheduleRunIdentifier for why. +// unit journald attributed it to; see scheduleRunIdentifier for why. A read +// that fails is left to fail: an empty history and an unreadable journal are +// different answers, and only one of them means the job never ran. func scheduleHistoryCommand(unit string, n int) string { if n <= 0 { n = 20 } return "journalctl SYSLOG_IDENTIFIER=" + scheduleRunIdentifier + " ONEBOX_UNIT=" + q(unit) + - " -o cat -r -n " + strconv.Itoa(n) + " --no-pager 2>/dev/null || true" + " -o cat -r -n " + strconv.Itoa(n) + " --no-pager" } // parseScheduleRunRecords keeps the lines that decode and drops the rest: a @@ -100,6 +102,9 @@ func (e *Engine) ScheduleHistory(ctx context.Context, name string, n int) ([]Sch if err != nil { return nil, err } + if res.ExitCode != 0 { + return nil, fmt.Errorf("read run records of %s from the host journal (exit %d): %s", name, res.ExitCode, strings.TrimSpace(res.Stderr)) + } records := parseScheduleRunRecords(res.Stdout) if records == nil { records = []ScheduleRunRecord{} diff --git a/internal/engine/schedule_history_test.go b/internal/engine/schedule_history_test.go index 8418209..a71927a 100644 --- a/internal/engine/schedule_history_test.go +++ b/internal/engine/schedule_history_test.go @@ -122,3 +122,17 @@ func TestScheduleLogsRefusesAJobWithoutRecords(t *testing.T) { } } } + +func TestScheduleHistorySurfacesAJournalReadFailure(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "journalctl") { + return transport.Result{ExitCode: 1, Stderr: "Failed to open journal: Permission denied"}, true + } + return transport.Result{}, false + } + _, err := e.ScheduleHistory(context.Background(), "nightly", 20) + if err == nil || !strings.Contains(err.Error(), "Permission denied") { + t.Fatalf("an unreadable journal was reported as an empty history: %v", err) + } +} diff --git a/internal/engine/schedule_run.go b/internal/engine/schedule_run.go index c4c6696..ba0210c 100644 --- a/internal/engine/schedule_run.go +++ b/internal/engine/schedule_run.go @@ -81,15 +81,22 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu } // noclobber: a second manual run before the first is consumed would - // otherwise rewrite the file under it and misattribute the inputs. + // otherwise rewrite the file under it and misattribute the inputs. The + // existence check in front gives that case its own exit status, so a + // host that simply refuses the write is reported as that and not as a + // pending run nobody can find. path := e.names().ScheduledJobRunInputs(name) - create := "umask 077 && install -d -m 700 " + q(e.names().AppDir()+"/schedule") + " && set -C && cat > " + q(path) + create := "if [ -e " + q(path) + " ]; then exit 73; fi; " + + "umask 077 && install -d -m 700 " + q(e.names().AppDir()+"/schedule") + " && set -C && cat > " + q(path) res, err := e.T.RunInput(ctx, create, scheduleInputsFile(operationID, inputs)) if err != nil { return result, err } - if res.ExitCode != 0 { + switch { + case res.ExitCode == 73: return result, fmt.Errorf("a manual run of %s is already pending (%s exists); wait for it, or remove the file on the host", name, path) + case res.ExitCode != 0: + return result, fmt.Errorf("cannot write the inputs file %s on the host: %s", path, strings.TrimSpace(res.Stderr)) } // From here on the file is ours to clean up: a request that fails before // the unit starts must not leave it behind to refuse the next one. @@ -170,7 +177,7 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu // the read. Matching on the operation id means a record left by an earlier // run, or by a timer firing that took this slot, is never reported as ours. func (e *Engine) awaitScheduleRecord(ctx context.Context, name, operationID string) (*ScheduleRunRecord, error) { - for attempt := 0; attempt < 10; attempt++ { + for attempt := range 10 { if attempt > 0 { e.Opts.Sleep(200 * time.Millisecond) } diff --git a/internal/engine/schedule_run_test.go b/internal/engine/schedule_run_test.go index 2feade0..137f5e5 100644 --- a/internal/engine/schedule_run_test.go +++ b/internal/engine/schedule_run_test.go @@ -183,3 +183,46 @@ func TestScheduleRunDiscardsItsInputsWhenTheStartFails(t *testing.T) { t.Fatalf("a failed start left the inputs file pending:\n%s", seq) } } + +func TestScheduleRunTellsAPendingFileFromAWriteFailure(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + for name, tc := range map[string]struct { + exit int + stderr string + want string + reject string + }{ + "pending": {73, "", "already pending", "cannot write"}, + "read-only": {1, "cat: cannot create: Read-only file system", "Read-only file system", "already pending"}, + } { + t.Run(name, func(t *testing.T) { + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "sync.inputs"): + return transport.Result{ExitCode: tc.exit, Stderr: tc.stderr}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + _, err := e.ScheduleRun(context.Background(), "op", "sync", nil, false) + if err == nil || !strings.Contains(err.Error(), tc.want) || strings.Contains(err.Error(), tc.reject) { + t.Fatalf("err = %v, want %q and not %q", err, tc.want, tc.reject) + } + for _, command := range f.Commands { + if strings.Contains(command, "systemctl start") { + t.Fatalf("a failed write still started the unit: %s", command) + } + } + }) + } +} diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index 2ccfc50..f79d939 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -1090,11 +1090,11 @@ func TestScheduledJobRunnerRetriesWithCappedDoublingBackoff(t *testing.T) { } } single := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) - if strings.Contains(single, "while :; do") { + if strings.Contains(single, "max_attempts=") { t.Errorf("a single-attempt job must not carry a retry loop:\n%s", single) } pinned := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "pinned", RetryAttempts: 2, RetryBackoff: time.Second, RetryMaxBackoff: time.Minute}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) - if !strings.Contains(pinned, "while :; do") || strings.Index(pinned, "flock --unlock 8") > strings.Index(pinned, "while :; do") { + if !strings.Contains(pinned, "max_attempts=") || strings.Index(pinned, "flock --unlock 8") > strings.Index(pinned, "max_attempts=") { t.Errorf("pinned runner must release the schedule mutex before its attempt loop:\n%s", pinned) } for _, script := range []string{runner, single, pinned} { From e09ce677a2d203cdd5f1f5df6c58dbff3b2d1ad8 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 21:33:52 -0700 Subject: [PATCH 19/25] fix(schedule): act on the second local review - A skip never clears a failure: ob status reports the newest run that actually happened and says nothing has run since. - An unreadable journal costs the status report its records, not the whole report; ob schedule history is still the command that says why. - A skipped run is notified as a run that did not happen, not as a failure. notify.Payload carries the distinction, and its reason survives redaction because it comes from Onebox's own closed vocabulary. - ob schedule run --wait only lets go of its inputs file once a record proves the runner read it, so a start that never activated, or one that merged into a timer firing, cannot strand the file. - The inputs file is written through the fence guard, like every other mutation. Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule.go | 55 +++++++++------ internal/engine/schedule_run.go | 23 ++++--- internal/engine/schedule_status.go | 33 ++++++--- internal/engine/schedule_test.go | 68 +++++++++++++++++-- internal/notify/notify.go | 25 +++++-- .../content/docs/guides/schedule-a-job.mdx | 5 ++ 6 files changed, 159 insertions(+), 50 deletions(-) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 3e39a61..2019392 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -523,32 +523,38 @@ func (e *Engine) scheduleNotifier(job app.ScheduledJob) (string, error) { } lines = append(lines, scheduleRunRecordLines(e.Spec.Name, e.names().ScheduledJobUnit(job.Name), job.Name, e.names().ScheduledJobRunState(job.Name))...) lines = append(lines, `case " `+strings.Join(job.Notify, " ")+` " in *" $outcome "*) ;; *) exit 0 ;; esac`) - wantsSuccess, wantsFailure := false, false + // Three classes, because a skip is neither: the job did not fail, and it + // did not do its work either. + var wants = map[string]bool{} for _, outcome := range job.Notify { - if outcome == "success" { - wantsSuccess = true - } else { - wantsFailure = true - } - } - var success, failure []string - var err error - if wantsSuccess { - if success, err = e.scheduleNotificationSends(job.Name, environment, "ok"); err != nil { - return "", err + switch outcome { + case "success": + wants["ok"] = true + case "skipped": + wants["skipped"] = true + default: + wants["fail"] = true + } + } + sends := map[string][]string{} + for _, class := range []string{"ok", "skipped", "fail"} { + if !wants[class] { + continue } - } - if wantsFailure { - if failure, err = e.scheduleNotificationSends(job.Name, environment, "fail"); err != nil { + rendered, err := e.scheduleNotificationSends(job.Name, environment, class) + if err != nil { return "", err } + sends[class] = rendered } - if len(success)+len(failure) > 0 { + if len(sends["ok"])+len(sends["skipped"])+len(sends["fail"]) > 0 { lines = append(lines, `ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ')`) lines = append(lines, `if [ "$outcome" = success ]; then`) - lines = append(lines, orNoop(success)...) + lines = append(lines, orNoop(sends["ok"])...) + lines = append(lines, `elif [ "$outcome" = skipped ]; then`) + lines = append(lines, orNoop(sends["skipped"])...) lines = append(lines, "else") - lines = append(lines, orNoop(failure)...) + lines = append(lines, orNoop(sends["fail"])...) lines = append(lines, "fi", "wait || true") } lines = append(lines, "exit 0", "") @@ -564,16 +570,23 @@ func orNoop(lines []string) []string { } // scheduleNotificationSends renders one backgrounded curl per notification -// that selects the given status. Status is the notify package's word: ok or -// fail. A failed send is logged and never replaces the job's own result. -func (e *Engine) scheduleNotificationSends(job, environment, status string) ([]string, error) { +// that selects the given class: ok, skipped, or fail. A skip routes with the +// failures, because that is the channel an operator watches, and says what it +// is rather than claiming the job failed. A failed send is logged and never +// replaces the job's own result. +func (e *Engine) scheduleNotificationSends(job, environment, class string) ([]string, error) { var sends []string for _, name := range sortedNames(e.Spec.Notifications) { cfg := e.Spec.Notifications[name] + status := "fail" + if class == "ok" { + status = "ok" + } payload := notify.Payload{ App: e.Spec.Name, Env: environment, Host: e.T.Destination(), Verb: "scheduled job " + job, Status: status, DeployID: scheduleNotificationRun, TS: scheduleNotificationTimestamp, + Skipped: class == "skipped", } if status != "ok" { payload.Error = "scheduled job failed; inspect trusted host diagnostics" diff --git a/internal/engine/schedule_run.go b/internal/engine/schedule_run.go index ba0210c..f071401 100644 --- a/internal/engine/schedule_run.go +++ b/internal/engine/schedule_run.go @@ -88,7 +88,7 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu path := e.names().ScheduledJobRunInputs(name) create := "if [ -e " + q(path) + " ]; then exit 73; fi; " + "umask 077 && install -d -m 700 " + q(e.names().AppDir()+"/schedule") + " && set -C && cat > " + q(path) - res, err := e.T.RunInput(ctx, create, scheduleInputsFile(operationID, inputs)) + res, err := e.mutateInput(ctx, create, scheduleInputsFile(operationID, inputs)) if err != nil { return result, err } @@ -137,17 +137,22 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu if err != nil { return result, err } - if res.ExitCode != 0 && !wait { - return result, fmt.Errorf("systemctl start %s: %s", unit, strings.TrimSpace(res.Stderr)) - } - // The unit was activated, so the runner owns the file now, whether it ran - // or skipped; a blocking start that exits non-zero still activated it. - pending = false - result.Started = true if !wait { + if res.ExitCode != 0 { + return result, fmt.Errorf("systemctl start %s: %s", unit, strings.TrimSpace(res.Stderr)) + } + // The unit is queued, so the runner owns the file now. + pending = false + result.Started = true e.logf("schedule: %s started as %s; ob schedule history %s shows the outcome", name, operationID, name) return result, nil } + // A blocking start that exits non-zero may mean the job failed, which is + // an outcome, or that the unit never activated, which is not. Only the + // record settles it, and only a record carrying this operation says the + // runner read the inputs file: a start that merged into a timer firing + // already in progress leaves that file untouched, for the next manual run + // that would otherwise be refused as pending. last, err := e.awaitScheduleRecord(ctx, name, operationID) if err != nil { if res.ExitCode != 0 { @@ -155,6 +160,8 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu } return result, err } + pending = false + result.Started = true result.Record = last exit := "-" if last.ExitStatus != nil { diff --git a/internal/engine/schedule_status.go b/internal/engine/schedule_status.go index 514a545..1a2d311 100644 --- a/internal/engine/schedule_status.go +++ b/internal/engine/schedule_status.go @@ -78,7 +78,10 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) "printf '%s\\n' "+q("@@"+job.Name+":run"), "cat "+q(e.names().ScheduledJobRunState(job.Name))+" 2>/dev/null || true", "printf '%s\\n' "+q("@@"+job.Name+":history"), - scheduleHistoryCommand(unit, 20), + // Status degrades rather than fails: an unreadable journal costs + // this section its records, not the whole report. `ob schedule + // history` is the command that says why the read failed. + scheduleHistoryCommand(unit, 20)+" 2>/dev/null || true", ) } res, err := e.T.Run(ctx, strings.Join(commands, "\n")) @@ -162,25 +165,29 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) } } } + // lastRun is the newest record that actually ran: the one whose + // outcome is the job's standing verdict. A skip is news about timing, + // and it must not clear a failure that nothing has fixed yet. + var lastRun *ScheduleRunRecord if len(records) > 0 { last := records[0] status.LastOutcome = last.Outcome status.LastReason = last.Reason status.LastDurationSeconds = last.DurationSeconds status.LastAttempts = last.Attempts - // Skips say nothing about the job, so they neither break nor - // extend a failure streak; their own streak is counted from the - // newest record until something actually ran. for _, record := range records { if record.Outcome != "skipped" { break } status.ConsecutiveSkips++ } - for _, record := range records { + for i, record := range records { if record.Outcome == "skipped" { continue } + if lastRun == nil { + lastRun = &records[i] + } if record.Outcome != "failure" && record.Outcome != "timeout" { break } @@ -193,14 +200,18 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) if service.loadState != "loaded" { status.Issues = append(status.Issues, "service unit is not loaded") } - // The record is the verdict: a recorded failure or timeout is an - // issue, and so is a job that keeps being skipped. - if status.LastOutcome == "failure" || status.LastOutcome == "timeout" { + // The record is the verdict: the newest run that actually happened is + // the one that counts, and so is a job that keeps being skipped. + if lastRun != nil && (lastRun.Outcome == "failure" || lastRun.Outcome == "timeout") { exit := "?" - if records[0].ExitStatus != nil { - exit = strconv.Itoa(*records[0].ExitStatus) + if lastRun.ExitStatus != nil { + exit = strconv.Itoa(*lastRun.ExitStatus) + } + issue := fmt.Sprintf("last run failed: %s (exit %s)", lastRun.Outcome, exit) + if status.LastOutcome == "skipped" { + issue += ", and nothing has run since" } - status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %s)", status.LastOutcome, exit)) + status.Issues = append(status.Issues, issue) } if status.ConsecutiveSkips >= skipStreakIssue { status.Issues = append(status.Issues, fmt.Sprintf("skipped %d firings in a row: %s", status.ConsecutiveSkips, status.LastReason)) diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index f79d939..c2bf009 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -1022,10 +1022,10 @@ NextElapseUSecRealtime=Sat 2026-09-06 02:00:00 UTC if got.LastOutcome != "skipped" || got.NextRun != "Sat 2026-09-06 02:00:00 UTC" || !got.JournalPersistent { t.Fatalf("record fields not surfaced: %#v", got) } - // The newest record is a skip: it neither counts as a failure nor clears - // the failure before it, and it raises no issue of its own. - if got.ConsecutiveFailures != 1 || got.Diverged { - t.Fatalf("a skip neither counts as nor clears a failure: %#v", got) + // The newest record is a skip: it is not itself a failure, and it does not + // clear the failure before it, which is still the job's standing verdict. + if got.ConsecutiveFailures != 1 || !got.Diverged { + t.Fatalf("a skip cleared the failure behind it: %#v", got) } if got.LastAttempts != 0 || got.LastDurationSeconds != 1 { t.Fatalf("last run detail not surfaced: %#v", got) @@ -1357,7 +1357,7 @@ func TestScheduleStatusRaisesAnIssueForASkipStreak(t *testing.T) { t.Fatal(err) } got := statuses[0] - if got.ConsecutiveSkips != tc.skips || got.ConsecutiveFailures != 0 || got.LastOutcome != "skipped" || got.LastReason == "" { + if got.ConsecutiveSkips != tc.skips || got.LastOutcome != "skipped" || got.LastReason == "" { t.Fatalf("skips not counted: %#v", got) } if got.Diverged != tc.issue { @@ -1369,3 +1369,61 @@ func TestScheduleStatusRaisesAnIssueForASkipStreak(t *testing.T) { }) } } + +// A skip is news about timing. It must not clear a failure that nothing has +// fixed: the newest record that actually ran is the standing verdict. +func TestScheduleStatusKeepsAFailureVisibleBehindASkip(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + history := `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:01Z","duration_s":0,"attempts":0,"exit_status":0,"outcome":"skipped","reason":"an application operation holds the deploy lock","inputs":{}} +{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"timer","started_at":"2026-09-04T02:00:01Z","finished_at":"2026-09-04T02:05:02Z","duration_s":301,"attempts":2,"exit_status":9,"outcome":"failure","inputs":{}} +{"run":"c3d4e5f60718293a4b5c6d7e8f901234","job":"nightly","trigger":"timer","started_at":"2026-09-03T02:00:01Z","finished_at":"2026-09-03T02:01:02Z","duration_s":61,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}}` + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: "@@journal\npersistent\n@@nightly:service\nLoadState=loaded\nActiveState=inactive\n@@nightly:timer\nLoadState=loaded\nActiveState=active\n@@nightly:run\n@@nightly:history\n" + history + "\n"}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + got := statuses[0] + if !got.Diverged || got.ConsecutiveFailures != 1 || got.LastOutcome != "skipped" { + t.Fatalf("a skip cleared the failure behind it: %#v", got) + } + if issues := strings.Join(got.Issues, "; "); !strings.Contains(issues, "last run failed: failure (exit 9)") || + !strings.Contains(issues, "nothing has run since") { + t.Fatalf("issue does not name the failure the skip hid: %#v", got.Issues) + } +} + +// An unreadable journal costs status its records, not the whole report. +func TestScheduleStatusDegradesWhenTheJournalCannotBeRead(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + if !strings.Contains(cmd, "2>/dev/null || true") { + return transport.Result{ExitCode: 1, Stderr: "Failed to open journal"}, true + } + return transport.Result{Stdout: "@@journal\npersistent\n@@nightly:service\nLoadState=loaded\nActiveState=inactive\n@@nightly:timer\nLoadState=loaded\nActiveState=active\n@@nightly:run\n@@nightly:history\n"}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatalf("an unreadable journal broke the whole status read: %v", err) + } + if got := statuses[0]; got.LastOutcome != "" || got.Diverged { + t.Fatalf("missing records must read as unknown, not as failure: %#v", got) + } +} diff --git a/internal/notify/notify.go b/internal/notify/notify.go index 1dff3f0..2665228 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -34,7 +34,13 @@ type Payload struct { Error string `json:"error,omitempty"` Operator string `json:"operator,omitempty"` TS string `json:"ts"` - Text string `json:"text"` // human line, filled by Send + // Skipped marks work that did not run rather than work that failed. It + // routes with the failures, because that is where an operator watches, + // but it must not be announced as one: a scheduled job that stood aside + // for a deploy did nothing wrong, and saying otherwise trains people to + // ignore the channel. + Skipped bool `json:"skipped,omitempty"` + Text string `json:"text"` // human line, filled by Send } // Request is the stable HTTP representation of one selected notification. @@ -55,13 +61,16 @@ func (p Payload) event() string { } func (p Payload) text() string { + id := p.DeployID + if id != "" { + id = " " + id + } if p.Status == "ok" { - id := p.DeployID - if id != "" { - id = " " + id - } return fmt.Sprintf("✅ %s: %s%s succeeded on %s", p.App, p.Verb, id, p.Host) } + if p.Skipped { + return fmt.Sprintf("⏭️ %s: %s%s did not run on %s — %s", p.App, p.Verb, id, p.Host, p.Error) + } return fmt.Sprintf("🚨 %s: %s FAILED on %s — %s", p.App, p.Verb, p.Host, p.Error) } @@ -89,6 +98,12 @@ func Prepare(cfg app.Notification, p Payload) (*Request, error) { // outcome; operators use the trusted local diagnostics for details. if p.Status != "ok" && p.Error != "" { p.Error = "operation failed; inspect trusted local diagnostics" + if p.Skipped { + // A skip's reason is Onebox's own closed vocabulary — a lock was + // held, a run was already going — with no provider output in it, + // so the sentence that makes the notification useful survives. + p.Error = "the run was skipped; inspect trusted host diagnostics" + } } p.Text = p.text() contentType := "application/json" diff --git a/site/src/content/docs/guides/schedule-a-job.mdx b/site/src/content/docs/guides/schedule-a-job.mdx index 5a8f439..782624d 100644 --- a/site/src/content/docs/guides/schedule-a-job.mdx +++ b/site/src/content/docs/guides/schedule-a-job.mdx @@ -152,6 +152,11 @@ status is never mistaken for a skip. Skips are recorded because a job that is silently never running looks exactly like one that works: one skip is timing, and three in a row are reported by `ob status` as an issue. +A skip is news about timing, not about the job, so it never clears a failure: +`ob status` keeps reporting the newest run that actually happened, and says +that nothing has run since. A job that selects `skipped` in `notify` is told +its run did not happen, not that it failed. + Read the records back from the workstation: ```sh From 5bf92a16c7210baca77cac1ce3f3806053e26fe3 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 22:20:41 -0700 Subject: [PATCH 20/25] fix(schedule): act on the second Copilot review - A record has to be one: it names the job, carries a systemd invocation id, and reports an outcome from the closed set. Callers read the newest record as the job's verdict, so a stray JSON line logged under the same fields could otherwise clear a failure. - The skip notification says where to look, once. The line already says the run did not happen. Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule_history.go | 19 +++++++++++++++---- internal/engine/schedule_history_test.go | 22 +++++++++++++++++++++- internal/engine/schedule_status.go | 2 +- internal/notify/notify.go | 7 +++---- 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/internal/engine/schedule_history.go b/internal/engine/schedule_history.go index f6d86dc..c25fa6d 100644 --- a/internal/engine/schedule_history.go +++ b/internal/engine/schedule_history.go @@ -61,9 +61,17 @@ func scheduleHistoryCommand(unit string, n int) string { " -o cat -r -n " + strconv.Itoa(n) + " --no-pager" } -// parseScheduleRunRecords keeps the lines that decode and drops the rest: a -// truncated or hand-written entry must not hide the records around it. -func parseScheduleRunRecords(stdout string) []ScheduleRunRecord { +// scheduleOutcomes is the closed vocabulary the notifier writes. A record +// claiming anything else did not come from a runner this binary generated. +var scheduleOutcomes = map[string]bool{"success": true, "failure": true, "timeout": true, "skipped": true} + +// parseScheduleRunRecords keeps the lines that are records of this job's runs +// and drops the rest. Decoding is not enough on its own: callers read the +// first entry as the newest run and let it stand for the job's health, so a +// stray JSON line that happened to be logged under the same identifier could +// clear a failure. A record has to name this job, carry a systemd invocation +// id, and report an outcome from the closed set before it counts. +func parseScheduleRunRecords(stdout, job string) []ScheduleRunRecord { var out []ScheduleRunRecord for _, line := range strings.Split(stdout, "\n") { line = strings.TrimSpace(line) @@ -74,6 +82,9 @@ func parseScheduleRunRecords(stdout string) []ScheduleRunRecord { if err := json.Unmarshal([]byte(line), &record); err != nil { continue } + if record.Job != job || !scheduleRunID.MatchString(record.Run) || !scheduleOutcomes[record.Outcome] { + continue + } out = append(out, record) } return out @@ -105,7 +116,7 @@ func (e *Engine) ScheduleHistory(ctx context.Context, name string, n int) ([]Sch if res.ExitCode != 0 { return nil, fmt.Errorf("read run records of %s from the host journal (exit %d): %s", name, res.ExitCode, strings.TrimSpace(res.Stderr)) } - records := parseScheduleRunRecords(res.Stdout) + records := parseScheduleRunRecords(res.Stdout, job.Name) if records == nil { records = []ScheduleRunRecord{} } diff --git a/internal/engine/schedule_history_test.go b/internal/engine/schedule_history_test.go index a71927a..f31ea24 100644 --- a/internal/engine/schedule_history_test.go +++ b/internal/engine/schedule_history_test.go @@ -16,7 +16,7 @@ not json ` func TestParseScheduleRunRecordsSkipsNoiseAndKeepsOrder(t *testing.T) { - records := parseScheduleRunRecords(sampleRunRecords) + records := parseScheduleRunRecords(sampleRunRecords, "nightly") if len(records) != 2 { t.Fatalf("records = %#v", records) } @@ -136,3 +136,23 @@ func TestScheduleHistorySurfacesAJournalReadFailure(t *testing.T) { t.Fatalf("an unreadable journal was reported as an empty history: %v", err) } } + +// Callers read the newest record as the job's standing verdict, so a line that +// merely happens to be JSON must not become one. +func TestParseScheduleRunRecordsRejectsLinesThatAreNotThisJobsRuns(t *testing.T) { + good := `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:02Z","duration_s":1,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}}` + for name, line := range map[string]string{ + "another job": `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"other","outcome":"success"}`, + "no run id": `{"run":"","job":"nightly","outcome":"success"}`, + "short run id": `{"run":"a1b2","job":"nightly","outcome":"success"}`, + "unknown outcome": `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","outcome":"cancelled"}`, + "unrelated json": `{"level":"info","msg":"something else"}`, + } { + t.Run(name, func(t *testing.T) { + records := parseScheduleRunRecords(line+"\n"+good, "nightly") + if len(records) != 1 || records[0].Outcome != "success" { + t.Fatalf("a line that is not this job's run was kept: %#v", records) + } + }) + } +} diff --git a/internal/engine/schedule_status.go b/internal/engine/schedule_status.go index 1a2d311..dac4667 100644 --- a/internal/engine/schedule_status.go +++ b/internal/engine/schedule_status.go @@ -108,7 +108,7 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) loadState: values["LoadState"], activeState: values["ActiveState"], release: values["release"], startedAt: values["started_at"], attempt: values["attempt"], next: values["NextElapseUSecRealtime"], - history: parseScheduleRunRecords(strings.Join(raw, "\n")), + history: parseScheduleRunRecords(strings.Join(raw, "\n"), name), } values = map[string]string{} raw = nil diff --git a/internal/notify/notify.go b/internal/notify/notify.go index 2665228..df2fd64 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -99,10 +99,9 @@ func Prepare(cfg app.Notification, p Payload) (*Request, error) { if p.Status != "ok" && p.Error != "" { p.Error = "operation failed; inspect trusted local diagnostics" if p.Skipped { - // A skip's reason is Onebox's own closed vocabulary — a lock was - // held, a run was already going — with no provider output in it, - // so the sentence that makes the notification useful survives. - p.Error = "the run was skipped; inspect trusted host diagnostics" + // The line already says the run did not happen, so this is only + // where to look for why. + p.Error = "inspect trusted host diagnostics" } } p.Text = p.text() From ec1cdc9ee326d55016621adbae32598423a8176c Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 5 Sep 2026 22:32:13 -0700 Subject: [PATCH 21/25] fix(schedule): act on the third local review - A firing that cannot take the job lock stands aside without writing state: that file belongs to the run in flight, and overwriting it replaced a real outcome with this activation's skip. - The container is cleared before every attempt, not once before the loop, so a corpse from one attempt cannot fail all the ones after it. - The failure notification carries the run id, which is what a text webhook sends and the only handle on ob schedule logs --run. - The systemd 252 floor is scoped back to what needs it: jobs declaring inputs, and ob schedule run. A host on an older LTS keeps running its scheduled jobs, and its records say trigger unknown rather than guessing. - ob schedule list reports a failed timer read instead of a table of dashes, and ob schedule logs fails like its siblings. Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- cmd/ob/schedule.go | 6 +- internal/engine/engine.go | 12 +- internal/engine/schedule.go | 80 +++++++++--- internal/engine/schedule_history.go | 5 + internal/engine/schedule_run.go | 7 + internal/engine/schedule_run_test.go | 33 +++++ internal/engine/schedule_test.go | 122 ++++++++++++++++-- internal/notify/notify.go | 5 +- .../content/docs/guides/schedule-a-job.mdx | 17 ++- 9 files changed, 246 insertions(+), 41 deletions(-) diff --git a/cmd/ob/schedule.go b/cmd/ob/schedule.go index e440223..6549d9b 100644 --- a/cmd/ob/schedule.go +++ b/cmd/ob/schedule.go @@ -162,8 +162,10 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { } return stream.terminal(cliOutcomeSuccess, data, nil) } - _, err = e.ScheduleLogs(cmd.Context(), args[0], logsRun, cmd.OutOrStdout(), cmd.ErrOrStderr()) - return err + if _, err := e.ScheduleLogs(cmd.Context(), args[0], logsRun, cmd.OutOrStdout(), cmd.ErrOrStderr()); err != nil { + return writeStructuredCommandFailure(cmd, g, "schedule_logs_failed", "run logs could not be read", err) + } + return nil }, } logsCmd.Flags().StringVar(&logsRun, "run", "", "run id from ob schedule history; default the newest run") diff --git a/internal/engine/engine.go b/internal/engine/engine.go index d5e75f4..b667aaa 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -90,10 +90,14 @@ type Engine struct { // wal-g invocation needs to know and which cannot change mid-operation. flockProbed bool flockPresent bool - Compose *ctypes.Project - T transport.Transport - Opts Options - ui *ui.UI + // triggerUnitProbed/triggerUnitPresent cache whether the host's systemd + // tells a timer activation from a manual one (TRIGGER_UNIT, systemd 252). + triggerUnitProbed bool + triggerUnitPresent bool + Compose *ctypes.Project + T transport.Transport + Opts Options + ui *ui.UI // fenceVal is " " once WriteFence has stamped the host; // mutate() guards every mutating command with it. diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 2019392..24be96b 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -109,7 +109,7 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { if e.Spec.Runtime != nil { runtimeEnvFiles = e.Spec.Runtime.EnvFiles } - runner := scheduleRunnerScript(e.Spec.Name, job, n, e.lockPath(), runtimeEnvFiles, e.lockTTL()) + runner := scheduleRunnerScript(e.Spec.Name, job, n, e.lockPath(), runtimeEnvFiles, e.lockTTL(), e.hasTriggerUnit(ctx)) notifier, err := e.scheduleNotifier(job) if err != nil { return fmt.Errorf("job %s: cannot render its failure notifier: %w", job.Name, err) @@ -172,9 +172,9 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { // job explicitly opts into the narrower pinned-release contract. Pinned mode // meets the deploy acquirer briefly under schedule.lock, leases the resolved // release before releasing that rendezvous, then retains only its own job lock. -func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile, lockTTL time.Duration) string { +func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile, lockTTL time.Duration, triggerUnit bool) string { if job.DeployLock == "pinned" { - return pinnedScheduleRunnerScript(application, job, names, applicationLock, runtimeEnvFiles, lockTTL) + return pinnedScheduleRunnerScript(application, job, names, applicationLock, runtimeEnvFiles, lockTTL, triggerUnit) } container := names.Container(job.Name, 1) projectDir := q(names.CurrentLink()) @@ -201,13 +201,13 @@ func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Na "trap 'exit 130' 2", "trap 'exit 143' 15", ) - lines = append(lines, scheduleRunPreamble()...) - lines = append(lines, scheduleAttemptLoop(job, compose)...) + lines = append(lines, scheduleRunPreamble(triggerUnit)...) + lines = append(lines, scheduleAttemptLoop(job, compose, container)...) lines = append(lines, "") return strings.Join(lines, "\n") } -func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile, lockTTL time.Duration) string { +func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile, lockTTL time.Duration, triggerUnit bool) string { scheduleDir := names.AppDir() + "/schedule" container := names.Container(job.Name, 1) projectDir := `"$release_dir"` @@ -239,11 +239,11 @@ func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names "trap 'exit 130' 2", "trap 'exit 143' 15", ) - lines = append(lines, scheduleRunPreamble()...) + lines = append(lines, scheduleRunPreamble(triggerUnit)...) // The lease is held; the schedule mutex goes back before the first // attempt so a compatible deploy is not blocked through the backoff. lines = append(lines, "/usr/bin/flock --unlock 8") - lines = append(lines, scheduleAttemptLoop(job, compose)...) + lines = append(lines, scheduleAttemptLoop(job, compose, container)...) lines = append(lines, "") return strings.Join(lines, "\n") } @@ -271,9 +271,15 @@ func scheduleLockLines(names app.Names, job, applicationLock string, lockTTL tim "tmp=\"$state.$$\"", // The operation and inputs of a manual request are kept on the skip // record too, so `ob schedule run --wait` can find its own outcome. + // Writing the state requires holding the job lock: it is the run in + // flight that owns that file, and overwriting it would replace a real + // run's outcome with this one's skip. "skip() { umask 077; printf 'skipped=%s\\noperation=%s\\ninputs=%s\\n' \"$1\" \"$operation\" \"$inputs_json\" >\"$tmp\"; mv -f \"$tmp\" \"$state\"; echo \"onebox: skipped: $1\" >&2; exit 0; }", + // No lock, no state: the run already in flight will record itself, + // and its evidence is not this activation's to overwrite. + "stand_aside() { echo \"onebox: skipped: $1\" >&2; exit 0; }", "exec 9>" + q(names.ScheduledJobRunLock(job)), - "/usr/bin/flock --exclusive --nonblock 9 || skip 'another run of this job is still in progress'", + "/usr/bin/flock --exclusive --nonblock 9 || stand_aside 'another run of this job is still in progress'", "exec 8>" + q(names.ScheduleRunLock()), "/usr/bin/flock --exclusive --nonblock 8 || skip 'an application operation is taking its lock'", "if [ -e " + q(applicationLock) + " ] && [ \"$(" + lockAgeCmd(applicationLock) + ")\" -le " + strconv.Itoa(ttlSeconds) + " ]; then skip 'an application operation holds the deploy lock'; fi", @@ -296,15 +302,41 @@ func (e *Engine) requireScheduleHost(ctx context.Context, jobs []app.ScheduledJo if !e.hasFlock(ctx) { return errors.New("scheduled jobs require flock on the target so they cannot overlap deployments; install util-linux and deploy again") } + // Declared inputs are the one feature that cannot work without + // TRIGGER_UNIT: the runner would have to guess whether an activation is + // the operator's, and guessing wrong hands a timer firing the inputs a + // person meant for their own run. Everything else works on an older + // systemd, so a host that has run scheduled jobs for years keeps running + // them — it only records `unknown` where it cannot know the trigger. + if !needsTriggerUnit(jobs) || e.hasTriggerUnit(ctx) { + return nil + } + return fmt.Errorf("a job declares inputs, which need systemd 252 or newer on the host: without $TRIGGER_UNIT the runner cannot tell a timer firing from an operator's run") +} + +// hasTriggerUnit reports whether the host's systemd sets TRIGGER_UNIT on a +// timer activation, which systemd 252 introduced. +func (e *Engine) hasTriggerUnit(ctx context.Context) bool { + if e.triggerUnitProbed { + return e.triggerUnitPresent + } res, err := e.T.Run(ctx, "systemctl --version 2>/dev/null | head -1") + e.triggerUnitProbed = true if err != nil { - return err + return false } - if version, ok := systemdVersion(res.Stdout); !ok || version < 252 { - return fmt.Errorf("scheduled jobs need systemd 252 or newer on the host, which tells a timer firing from a manual start; the host reports %q", - strings.TrimSpace(res.Stdout)) + version, ok := systemdVersion(res.Stdout) + e.triggerUnitPresent = ok && version >= 252 + return e.triggerUnitPresent +} + +func needsTriggerUnit(jobs []app.ScheduledJob) bool { + for _, job := range jobs { + if len(job.Inputs) > 0 { + return true + } } - return nil + return false } func scheduleContainerCleanup(container string) string { @@ -329,11 +361,19 @@ func scheduleStateFunction() []string { // scheduleRunPreamble sets the variables write_state records. The trigger is // systemd's own word for it: a timer activation carries TRIGGER_UNIT (systemd // 252 and newer), anything else is an operator. -func scheduleRunPreamble() []string { +func scheduleRunPreamble(triggerUnit bool) []string { + // TRIGGER_UNIT is set on a timer activation and on nothing else, so its + // absence names an operator — but only on a systemd that sets it at all. + // On an older host the runner says `unknown` rather than inventing a + // trigger it cannot observe. + otherwise := "unknown" + if triggerUnit { + otherwise = "manual" + } return append([]string{ "started_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", "started_epoch=$(date -u '+%s')", - "if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=manual; fi", + "if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=" + otherwise + "; fi", }, scheduleStateFunction()...) } @@ -373,7 +413,7 @@ func systemdVersion(firstLine string) (int, bool) { // spent. Backoff doubles and is capped; every sleep happens under the locks // the run already holds, which is why validation keeps the sum under the // timeout. A single-attempt job gets no loop, so its runner reads as before. -func scheduleAttemptLoop(job app.ScheduledJob, compose string) []string { +func scheduleAttemptLoop(job app.ScheduledJob, compose, container string) []string { if job.RetryAttempts <= 1 { return []string{"write_state 1", compose} } @@ -385,6 +425,10 @@ func scheduleAttemptLoop(job app.ScheduledJob, compose string) []string { "attempt=1", "while :; do", " write_state \"$attempt\"", + // The container name is fixed, so a corpse from the previous attempt + // would fail every attempt after it with "name already in use" and + // turn one transient failure into all of them. + " " + scheduleContainerCleanup(container), " status=0", " " + compose + " || status=$?", " [ \"$status\" -eq 0 ] && exit 0", @@ -468,7 +512,7 @@ func scheduleRunRecordLines(application, unit, job, state string) []string { " done <\"$state\"", " rm -f \"$state\"", "fi", - "if [ -z \"$trigger\" ]; then if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=manual; fi; fi", + "if [ -z \"$trigger\" ]; then if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=unknown; fi; fi", "result=${SERVICE_RESULT:-success}", "status=${EXIT_STATUS:-0}", // EXIT_STATUS is a signal name when the main process was killed. diff --git a/internal/engine/schedule_history.go b/internal/engine/schedule_history.go index c25fa6d..25e3e41 100644 --- a/internal/engine/schedule_history.go +++ b/internal/engine/schedule_history.go @@ -143,6 +143,11 @@ func (e *Engine) ScheduleList(ctx context.Context) ([]ScheduleListing, error) { if err != nil { return nil, err } + if res.ExitCode != 0 { + // Blank timer state and "the host would not tell us" are different + // answers, and a table of dashes must not stand for the second. + return nil, fmt.Errorf("read scheduled-job timers (exit %d): %s", res.ExitCode, strings.TrimSpace(res.Stderr)) + } observed := map[string]map[string]string{} current := "" for _, line := range strings.Split(res.Stdout, "\n") { diff --git a/internal/engine/schedule_run.go b/internal/engine/schedule_run.go index f071401..eff62e8 100644 --- a/internal/engine/schedule_run.go +++ b/internal/engine/schedule_run.go @@ -52,6 +52,13 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu if err := app.ValidateJobInputValues(workload, inputs); err != nil { return result, err } + // The inputs file is addressed to this activation and to no other. Only a + // host whose systemd sets TRIGGER_UNIT lets the runner tell them apart; on + // an older one the next timer firing would read the file meant for this + // run, and the run itself would be recorded as a firing. + if !e.hasTriggerUnit(ctx) { + return result, errors.New("this host's systemd does not set $TRIGGER_UNIT, so a timer firing cannot be told from this run; ob schedule run needs systemd 252 or newer. The timer itself keeps working") + } unit := e.names().ScheduledJobUnit(name) result.Unit = unit diff --git a/internal/engine/schedule_run_test.go b/internal/engine/schedule_run_test.go index 137f5e5..863f866 100644 --- a/internal/engine/schedule_run_test.go +++ b/internal/engine/schedule_run_test.go @@ -226,3 +226,36 @@ func TestScheduleRunTellsAPendingFileFromAWriteFailure(t *testing.T) { }) } } + +// Without TRIGGER_UNIT the next timer firing would read the file this run +// left, so the manual path is refused rather than being made ambiguous. +func TestScheduleRunRefusesAHostThatCannotTellTheTriggerApart(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "systemctl --version"): + return transport.Result{Stdout: "systemd 249 (249.11-0ubuntu3)\n"}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + _, err := e.ScheduleRun(context.Background(), "op", "sync", nil, false) + if err == nil || !strings.Contains(err.Error(), "systemd 252") { + t.Fatalf("err = %v, want a refusal naming the requirement", err) + } + for _, command := range f.Commands { + if strings.Contains(command, ".inputs") || strings.Contains(command, "systemctl start") { + t.Fatalf("the refused run still touched the host: %s", command) + } + } +} diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index c2bf009..133a6ee 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -179,7 +179,7 @@ func TestScheduledJobUnitContract(t *testing.T) { Calendar: "*-*-* 02:00:00", Timeout: "45m", CatchUp: false, DeployLock: "exclusive", } names := app.Names{App: "sample", BasePath: "/var/lib/ob"} - runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) service := scheduleServiceUnit("sample", job, "/etc/systemd/system/ob-sample-nightly.run", "/etc/systemd/system/ob-sample-nightly.notify") @@ -187,7 +187,7 @@ func TestScheduledJobUnitContract(t *testing.T) { for _, want := range []string{ "exec 9>'/var/lib/ob/sample/schedule/nightly.lock'", - "flock --exclusive --nonblock 9 || skip", + "flock --exclusive --nonblock 9 || stand_aside", "exec 8>'/var/lib/ob/sample/schedule.lock'", "flock --exclusive --nonblock 8 || skip", "/var/lib/ob/sample/lock", @@ -241,11 +241,11 @@ func TestPinnedScheduledJobRunnerLeasesImmutableRelease(t *testing.T) { runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", []app.EnvFile{ {File: "config/runtime.env"}, {File: "secrets/runtime.env", Provider: "sops"}, - }, 10*time.Minute) + }, 10*time.Minute, true) for _, want := range []string{ "exec 9>'/var/lib/ob/sample/schedule/refresh.lock'", - "flock --exclusive --nonblock 9 || skip", + "flock --exclusive --nonblock 9 || stand_aside", "exec 8>'/var/lib/ob/sample/schedule.lock'", "release_dir=$(readlink -f '/var/lib/ob/sample/current')", "exec 7>>\"$release_dir/.ob-schedule.lease\"", @@ -341,7 +341,7 @@ func TestPinnedScheduledJobLockProtocol(t *testing.T) { } job := app.ScheduledJob{Name: "refresh", DeployLock: "pinned"} - runner := scheduleRunnerScript("sample", job, names, filepath.Join(names.AppDir(), "lock"), nil, 10*time.Minute) + runner := scheduleRunnerScript("sample", job, names, filepath.Join(names.AppDir(), "lock"), nil, 10*time.Minute, true) runner = strings.ReplaceAll(runner, "/usr/bin/docker", q(stub)) command := exec.CommandContext(ctx, "sh") command.Stdin = strings.NewReader(runner) @@ -803,7 +803,7 @@ func TestScheduledJobRunnersRecordRunStateForTheNotifier(t *testing.T) { {"pinned", app.ScheduledJob{Name: "nightly", Cron: "0 2 * * *", Timezone: "UTC", Calendar: "*-*-* 02:00:00", Timeout: "45m", DeployLock: "pinned"}}, } { t.Run(tc.name, func(t *testing.T) { - runner := scheduleRunnerScript("sample", tc.job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) + runner := scheduleRunnerScript("sample", tc.job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) for _, want := range []string{ "state='/var/lib/ob/sample/schedule/nightly.state'", "write_state() {", @@ -1077,7 +1077,7 @@ func TestScheduledJobRunnerRetriesWithCappedDoublingBackoff(t *testing.T) { job := app.ScheduledJob{Name: "nightly", Timeout: "45m", DeployLock: "exclusive", RetryAttempts: 3, RetryBackoff: 30 * time.Second, RetryMaxBackoff: 10 * time.Minute} names := app.Names{App: "sample", BasePath: "/var/lib/ob"} - runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) for _, want := range []string{ "max_attempts=3", "backoff=30", "max_backoff=600", "attempt=1", "while :; do", "write_state \"$attempt\"", @@ -1089,11 +1089,11 @@ func TestScheduledJobRunnerRetriesWithCappedDoublingBackoff(t *testing.T) { t.Errorf("runner is missing %q:\n%s", want, runner) } } - single := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) + single := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) if strings.Contains(single, "max_attempts=") { t.Errorf("a single-attempt job must not carry a retry loop:\n%s", single) } - pinned := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "pinned", RetryAttempts: 2, RetryBackoff: time.Second, RetryMaxBackoff: time.Minute}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) + pinned := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "pinned", RetryAttempts: 2, RetryBackoff: time.Second, RetryMaxBackoff: time.Minute}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) if !strings.Contains(pinned, "max_attempts=") || strings.Index(pinned, "flock --unlock 8") > strings.Index(pinned, "max_attempts=") { t.Errorf("pinned runner must release the schedule mutex before its attempt loop:\n%s", pinned) } @@ -1202,7 +1202,7 @@ func TestScheduledJobRunnerConsumesManualInputsWithoutShellInterpolation(t *test job := app.ScheduledJob{Name: "sync", Timeout: "45m", DeployLock: "pinned", RetryAttempts: 1, Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog"}, Default: "catalog"}}} names := app.Names{App: "sample", BasePath: "/var/lib/ob"} - runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) for _, want := range []string{ "inputs_file='/var/lib/ob/sample/schedule/sync.inputs'", `if [ -z "${TRIGGER_UNIT:-}" ] && [ -f "$inputs_file" ]; then`, @@ -1229,7 +1229,7 @@ func TestScheduledJobRunnerConsumesManualInputsWithoutShellInterpolation(t *test if strings.Index(runner, "inputs_file=") > strings.Index(runner, "flock --exclusive --nonblock 9") { t.Fatalf("inputs are consumed after the lock:\n%s", runner) } - exclusive := scheduleRunnerScript("sample", app.ScheduledJob{Name: "sync", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute) + exclusive := scheduleRunnerScript("sample", app.ScheduledJob{Name: "sync", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) if !strings.Contains(exclusive, "inputs_file=") || !strings.Contains(exclusive, `run --rm --no-deps "$@" --name`) { t.Fatalf("exclusive runner does not consume inputs:\n%s", exclusive) } @@ -1427,3 +1427,103 @@ func TestScheduleStatusDegradesWhenTheJournalCannotBeRead(t *testing.T) { t.Fatalf("missing records must read as unknown, not as failure: %#v", got) } } + +// A firing that cannot take the job lock must leave the running job's state +// alone: that file is the evidence its own notifier turns into the record. +func TestScheduledJobRunnerDoesNotClobberARunningJobsState(t *testing.T) { + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + job := app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + if !strings.Contains(runner, `stand_aside() { echo "onebox: skipped: $1" >&2; exit 0; }`) { + t.Fatalf("runner has no lock-less skip:\n%s", runner) + } + if !strings.Contains(runner, "flock --exclusive --nonblock 9 || stand_aside 'another run of this job is still in progress'") { + t.Fatalf("a job-lock conflict still writes state:\n%s", runner) + } + // The other two skips hold the job lock, so the state is theirs to write. + for _, want := range []string{ + "flock --exclusive --nonblock 8 || skip 'an application operation is taking its lock'", + "skip 'an application operation holds the deploy lock'", + } { + if !strings.Contains(runner, want) { + t.Fatalf("runner is missing %q:\n%s", want, runner) + } + } +} + +// The container name is fixed, so a corpse from one attempt would fail every +// attempt after it. +func TestScheduledJobRunnerClearsTheContainerBetweenAttempts(t *testing.T) { + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + job := app.ScheduledJob{Name: "nightly", Timeout: "45m", DeployLock: "exclusive", + RetryAttempts: 3, RetryBackoff: time.Second, RetryMaxBackoff: time.Minute} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + loop := runner[strings.Index(runner, "while :; do"):] + if !strings.Contains(loop, "docker rm -f 'sample-nightly-1'") { + t.Fatalf("no cleanup inside the attempt loop:\n%s", loop) + } +} + +// On a systemd without TRIGGER_UNIT the runner cannot see the trigger, and +// says so rather than calling every timer firing an operator's run. +func TestScheduledJobRunnerRecordsAnUnknownTriggerOnAnOlderSystemd(t *testing.T) { + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + job := app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1} + modern := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + older := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, false) + if !strings.Contains(modern, "else trigger=manual; fi") { + t.Fatalf("a host that sets TRIGGER_UNIT must name the operator:\n%s", modern) + } + if !strings.Contains(older, "else trigger=unknown; fi") { + t.Fatalf("a host without TRIGGER_UNIT must not invent a trigger:\n%s", older) + } +} + +// The systemd floor is scoped to what actually needs it. A project that has +// been running scheduled jobs on an older LTS keeps running them. +func TestSyncSchedulesRequiresSystemd252OnlyForInputs(t *testing.T) { + for name, tc := range map[string]struct { + inputs map[string]app.JobInput + wantErr bool + }{ + "plain job": {nil, false}, + "declares inputs": {map[string]app.JobInput{"SOURCE": {Enum: []string{"a"}, Default: "a"}}, true}, + } { + t.Run(name, func(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", Inputs: tc.inputs, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "list-unit-files"): + return transport.Result{}, true + case strings.Contains(cmd, "systemd-analyze calendar"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl --version"): + return transport.Result{Stdout: "systemd 249 (249.11-0ubuntu3)\n"}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + err := e.SyncSchedules(context.Background()) + if tc.wantErr { + if err == nil || !strings.Contains(err.Error(), "systemd 252") { + t.Fatalf("inputs were accepted on systemd 249: %v", err) + } + return + } + if err != nil { + t.Fatalf("a job without inputs was refused on systemd 249: %v", err) + } + if artifacts := strings.Join(f.Inputs, "\n"); !strings.Contains(artifacts, "else trigger=unknown; fi") { + t.Fatalf("the runner claims a trigger the host cannot report:\n%s", artifacts) + } + }) + } +} diff --git a/internal/notify/notify.go b/internal/notify/notify.go index df2fd64..86157fa 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -71,7 +71,10 @@ func (p Payload) text() string { if p.Skipped { return fmt.Sprintf("⏭️ %s: %s%s did not run on %s — %s", p.App, p.Verb, id, p.Host, p.Error) } - return fmt.Sprintf("🚨 %s: %s FAILED on %s — %s", p.App, p.Verb, p.Host, p.Error) + // The id belongs on the failure line most of all: a text webhook sends + // this sentence and nothing else, so without it the one notification an + // operator acts on cannot name the run they should go and read. + return fmt.Sprintf("🚨 %s: %s%s FAILED on %s — %s", p.App, p.Verb, id, p.Host, p.Error) } // Prepare renders a selected payload without sending it. An unset webhook and diff --git a/site/src/content/docs/guides/schedule-a-job.mdx b/site/src/content/docs/guides/schedule-a-job.mdx index 782624d..79087c4 100644 --- a/site/src/content/docs/guides/schedule-a-job.mdx +++ b/site/src/content/docs/guides/schedule-a-job.mdx @@ -278,11 +278,18 @@ gates of `ob job run` below. The runner tells a manual activation from a timer firing by the `TRIGGER_UNIT` variable systemd sets on timer activations, which that project introduced in -version 252. Every scheduled job therefore needs systemd 252 or newer on the -host, not only one with inputs: the record's `trigger` is part of the contract, -and on an older systemd a timer firing would both be recorded as manual and -consume an operator's pending inputs. `ob preflight` and `ob deploy` refuse an -older host before anything is staged. Ubuntu 24.04 and Debian 12 qualify. +version 252. Ubuntu 24.04 and Debian 12 qualify; Ubuntu 22.04 and Debian 11 do +not. + +Scheduled jobs still run on those older hosts, unchanged. Two things narrow: + +- A job that declares `inputs` is refused, by `ob preflight` and by `ob deploy`, + before anything is staged. Without `TRIGGER_UNIT` the next timer firing would + read the file meant for an operator's run. +- `ob schedule run` is refused for the same reason. The timer keeps firing. + +The records on such a host say `trigger: unknown`, because the runner cannot +observe what started it and will not guess. ## Running one by hand From d8d94e9c0eba5f3a47e6ea881cb8f5f0a063bdf6 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 6 Sep 2026 07:14:48 -0700 Subject: [PATCH 22/25] fix(schedule): record a stand-aside without touching the running job's state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A firing that could not take the job lock exited 0 and wrote nothing, so its own ExecStopPost saw a clean exit and recorded a success for a run that never happened — and read and removed the state file belonging to the run that was still going, destroying its outcome and its pinned-release evidence. It now leaves a note keyed to its own invocation. The notifier prefers that note, records the skip with its reason, and leaves the state file alone. ScheduleList no longer swallows its own probe failure, which had made the exit check added alongside it unreachable: a host that will not answer is reported rather than rendered as a row of dashes. Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule.go | 26 +++++++-- internal/engine/schedule_history.go | 5 +- internal/engine/schedule_history_test.go | 18 ++++++ internal/engine/schedule_test.go | 71 +++++++++++++++++++++--- 4 files changed, 107 insertions(+), 13 deletions(-) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 24be96b..f313974 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -275,9 +275,14 @@ func scheduleLockLines(names app.Names, job, applicationLock string, lockTTL tim // flight that owns that file, and overwriting it would replace a real // run's outcome with this one's skip. "skip() { umask 077; printf 'skipped=%s\\noperation=%s\\ninputs=%s\\n' \"$1\" \"$operation\" \"$inputs_json\" >\"$tmp\"; mv -f \"$tmp\" \"$state\"; echo \"onebox: skipped: $1\" >&2; exit 0; }", - // No lock, no state: the run already in flight will record itself, - // and its evidence is not this activation's to overwrite. - "stand_aside() { echo \"onebox: skipped: $1\" >&2; exit 0; }", + // No lock, so no claim on the state file: the run already in flight + // owns it and will record itself. This activation leaves its own note + // instead, keyed to its own invocation, and the notifier reads that + // rather than the state a different run is still writing. Without the + // note the notifier would see a clean exit and record this activation + // as a success that never ran. + "skip_marker=\"$state.skip.${INVOCATION_ID:-$$}\"", + "stand_aside() { umask 077; printf 'skipped=%s\\noperation=%s\\ninputs=%s\\n' \"$1\" \"$operation\" \"$inputs_json\" >\"$skip_marker\"; echo \"onebox: skipped: $1\" >&2; exit 0; }", "exec 9>" + q(names.ScheduledJobRunLock(job)), "/usr/bin/flock --exclusive --nonblock 9 || stand_aside 'another run of this job is still in progress'", "exec 8>" + q(names.ScheduleRunLock()), @@ -497,7 +502,20 @@ func scheduleRunRecordLines(application, unit, job, state string) []string { return []string{ "state=" + q(state), "release=''; started_at=''; started_epoch=''; trigger=''; operation=''; attempt=0; inputs=''; skipped=''", - "if [ -f \"$state\" ]; then", + // A run that stood aside left a note under its own invocation. It + // never held the job lock, so the state file belongs to whichever run + // is still going: read the note and leave that file alone. + "skip_marker=\"$state.skip.${INVOCATION_ID:-}\"", + "if [ -f \"$skip_marker\" ]; then", + " while IFS= read -r line || [ -n \"$line\" ]; do", + " case \"$line\" in", + " skipped=*) skipped=${line#skipped=} ;;", + " operation=*) operation=${line#operation=} ;;", + " inputs=*) inputs=${line#inputs=} ;;", + " esac", + " done <\"$skip_marker\"", + " rm -f \"$skip_marker\"", + "elif [ -f \"$state\" ]; then", " while IFS= read -r line || [ -n \"$line\" ]; do", " case \"$line\" in", " release=*) release=${line#release=} ;;", diff --git a/internal/engine/schedule_history.go b/internal/engine/schedule_history.go index 25e3e41..2ae4b53 100644 --- a/internal/engine/schedule_history.go +++ b/internal/engine/schedule_history.go @@ -137,7 +137,10 @@ func (e *Engine) ScheduleList(ctx context.Context) ([]ScheduleListing, error) { unit := e.names().ScheduledJobUnit(job.Name) commands = append(commands, "printf '%s\\n' "+q("@@"+job.Name), - "systemctl show "+q(unit+".timer")+" --no-pager --property=ActiveState --property=NextElapseUSecRealtime --property=LastTriggerUSec 2>/dev/null || true") + // Left to fail: the exit check below distinguishes a timer with + // nothing to say from a host that would not answer, and `|| true` + // would make that check unreachable. + "systemctl show "+q(unit+".timer")+" --no-pager --property=ActiveState --property=NextElapseUSecRealtime --property=LastTriggerUSec") } res, err := e.T.Run(ctx, strings.Join(commands, "\n")) if err != nil { diff --git a/internal/engine/schedule_history_test.go b/internal/engine/schedule_history_test.go index f31ea24..462c59a 100644 --- a/internal/engine/schedule_history_test.go +++ b/internal/engine/schedule_history_test.go @@ -156,3 +156,21 @@ func TestParseScheduleRunRecordsRejectsLinesThatAreNotThisJobsRuns(t *testing.T) }) } } + +// A timer read that fails is not a timer with nothing to say. Dashes in the +// table must not stand for a host that would not answer. +func TestScheduleListSurfacesAFailedTimerRead(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{ExitCode: 1, Stderr: "Failed to connect to bus"}, true + } + return transport.Result{}, false + } + if _, err := e.ScheduleList(context.Background()); err == nil || !strings.Contains(err.Error(), "Failed to connect to bus") { + t.Fatalf("an unreadable timer was reported as blank state: %v", err) + } + if seq := strings.Join(f.Commands, "\n"); strings.Contains(seq, "|| true") { + t.Fatalf("the read swallows its own failure:\n%s", seq) + } +} diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index 133a6ee..bf68036 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -873,11 +873,27 @@ func TestScheduledJobNotifierWritesOneRunRecordToTheJournal(t *testing.T) { // script wrote, whether the state file survived, and every curl invocation's // arguments, one per element. func runNotifier(t *testing.T, job app.ScheduledJob, notifications map[string]app.Notification, state string, env map[string]string) (map[string]any, bool, []string) { + t.Helper() + base := t.TempDir() + if state != "" { + scheduleDir := filepath.Join(base, "sample", "schedule") + if err := os.MkdirAll(scheduleDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(scheduleDir, "nightly.state"), []byte(state), 0o600); err != nil { + t.Fatal(err) + } + } + return runNotifierIn(t, base, job, notifications, env) +} + +// runNotifierIn is runNotifier over a directory the caller has already laid +// out, for the cases whose whole point is which files exist beforehand. +func runNotifierIn(t *testing.T, base string, job app.ScheduledJob, notifications map[string]app.Notification, env map[string]string) (map[string]any, bool, []string) { t.Helper() if runtime.GOOS == "windows" { t.Skip("POSIX shell required") } - base := t.TempDir() cfg := testConfig() cfg.BasePath = base cfg.Notifications = notifications @@ -891,11 +907,6 @@ func runNotifier(t *testing.T, job app.ScheduledJob, notifications map[string]ap t.Fatal(err) } statePath := filepath.Join(scheduleDir, "nightly.state") - if state != "" { - if err := os.WriteFile(statePath, []byte(state), 0o600); err != nil { - t.Fatal(err) - } - } bin := t.TempDir() record := filepath.Join(bin, "record.jsonl") // The stub checks the structured fields the history query relies on and @@ -1434,8 +1445,14 @@ func TestScheduledJobRunnerDoesNotClobberARunningJobsState(t *testing.T) { names := app.Names{App: "sample", BasePath: "/var/lib/ob"} job := app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1} runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) - if !strings.Contains(runner, `stand_aside() { echo "onebox: skipped: $1" >&2; exit 0; }`) { - t.Fatalf("runner has no lock-less skip:\n%s", runner) + // The note is keyed to this activation, so it cannot be mistaken for the + // state file of the run that holds the lock. + if !strings.Contains(runner, `skip_marker="$state.skip.${INVOCATION_ID:-$$}"`) || + !strings.Contains(runner, `stand_aside() { umask 077; printf 'skipped=%s\noperation=%s\ninputs=%s\n' "$1" "$operation" "$inputs_json" >"$skip_marker"`) { + t.Fatalf("runner has no lock-less skip note:\n%s", runner) + } + if strings.Contains(runner, `stand_aside`) && strings.Contains(runner, `>"$state"; echo "onebox: skipped`) { + t.Fatalf("the lock-less skip writes the running job's state:\n%s", runner) } if !strings.Contains(runner, "flock --exclusive --nonblock 9 || stand_aside 'another run of this job is still in progress'") { t.Fatalf("a job-lock conflict still writes state:\n%s", runner) @@ -1527,3 +1544,41 @@ func TestSyncSchedulesRequiresSystemd252OnlyForInputs(t *testing.T) { }) } } + +// An activation that stood aside must record itself as skipped without +// touching the state file the running job is still writing. Getting this +// wrong records a run that never happened as a success, and destroys the +// evidence of the one that did. +func TestScheduledJobNotifierReadsAStandAsideNoteAndSpareTheRunningState(t *testing.T) { + base := t.TempDir() + scheduleDir := filepath.Join(base, "sample", "schedule") + if err := os.MkdirAll(scheduleDir, 0o700); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(scheduleDir, "nightly.state") + liveState := "release=r1\nstarted_at=2026-09-05T15:00:01Z\nstarted_epoch=1\ntrigger=timer\noperation=\nattempt=2\ninputs=\n" + if err := os.WriteFile(statePath, []byte(liveState), 0o600); err != nil { + t.Fatal(err) + } + marker := statePath + ".skip.abc123" + if err := os.WriteFile(marker, []byte("skipped=another run of this job is still in progress\noperation=op-7\ninputs=\n"), 0o600); err != nil { + t.Fatal(err) + } + + record, _, _ := runNotifierIn(t, base, app.ScheduledJob{Name: "nightly", Notify: []string{"failure", "timeout"}}, nil, map[string]string{ + "SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "abc123", + }) + if record["outcome"] != "skipped" || record["reason"] != "another run of this job is still in progress" { + t.Fatalf("a stand-aside activation was not recorded as skipped: %#v", record) + } + if record["operation"] != "op-7" { + t.Fatalf("the note's operation was lost: %#v", record) + } + body, err := os.ReadFile(statePath) + if err != nil || string(body) != liveState { + t.Fatalf("the running job's state was disturbed: %v %q", err, string(body)) + } + if _, err := os.Stat(marker); err == nil { + t.Fatal("the stand-aside note survived its own notifier") + } +} From bcac16178d0562faecd8cee4605ffabb31811560 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 6 Sep 2026 07:25:18 -0700 Subject: [PATCH 23/25] fix(schema): give a list field a list default A JSON Schema default has to be a value of the property's own type. Every list field carried its default as the sentence the reference table prints, `failure, timeout`, so the published schema declared an array whose default was a string. An editor that applies defaults would have filled the list with that sentence. The same shape was in every list `examples` entry. Fixed where it was produced rather than per field, so `schedule.notify` and the older `notifications.on` are both correct, and the reference table still reads the way it did: the generator splits the tag, the docs join it back. Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- cmd/ob-docgen/main.go | 11 +++++ docs/onebox.run-v1.schema.json | 18 ++++++-- internal/app/jsonschema.go | 19 ++++++++- internal/app/jsonschema_test.go | 60 +++++++++++++++++++++++++++ site/public/onebox.run-v1.schema.json | 18 ++++++-- 5 files changed, 117 insertions(+), 9 deletions(-) diff --git a/cmd/ob-docgen/main.go b/cmd/ob-docgen/main.go index 118ffc0..9ad4bf5 100644 --- a/cmd/ob-docgen/main.go +++ b/cmd/ob-docgen/main.go @@ -636,6 +636,17 @@ func defaultOf(node map[string]any) string { if !ok { return "" } + // A list default is an array in the schema, because that is the only + // thing a JSON Schema default for an array property may be. Printing it + // with Go's slice syntax puts `[failure timeout]` in a reference table + // nobody writes that way; the table reads as the value would be written. + if items, ok := v.([]any); ok { + parts := make([]string, 0, len(items)) + for _, item := range items { + parts = append(parts, fmt.Sprint(item)) + } + return strings.Join(parts, ", ") + } return fmt.Sprint(v) } diff --git a/docs/onebox.run-v1.schema.json b/docs/onebox.run-v1.schema.json index 76ba0e3..6c76f12 100644 --- a/docs/onebox.run-v1.schema.json +++ b/docs/onebox.run-v1.schema.json @@ -731,7 +731,9 @@ "backup_key_material": { "description": "Key-material identities the backup report must name.", "examples": [ - "BACKUP_ACCESS_KEY_ID" + [ + "BACKUP_ACCESS_KEY_ID" + ] ], "items": { "type": "string" @@ -1123,7 +1125,10 @@ "type": "string" }, "on": { - "default": "success, failure", + "default": [ + "success", + "failure" + ], "description": "Operation outcomes that trigger this notification.", "items": { "enum": [ @@ -2266,7 +2271,9 @@ "enum": { "description": "Accepted values.", "examples": [ - "catalog" + [ + "catalog" + ] ], "items": { "type": "string" @@ -2598,7 +2605,10 @@ "type": "string" }, "notify": { - "default": "failure, timeout", + "default": [ + "failure", + "timeout" + ], "description": "Run outcomes that send the configured notifications: success, failure, timeout, skipped.", "items": { "enum": [ diff --git a/internal/app/jsonschema.go b/internal/app/jsonschema.go index 920d4f1..5a36027 100644 --- a/internal/app/jsonschema.go +++ b/internal/app/jsonschema.go @@ -129,7 +129,8 @@ func annotateSchemaField(schema map[string]any, field reflect.StructField) { } func schemaTagValue(value string, t reflect.Type) any { - switch deref(t).Kind() { + target := deref(t) + switch target.Kind() { case reflect.Bool: if parsed, err := strconv.ParseBool(value); err == nil { return parsed @@ -142,6 +143,22 @@ func schemaTagValue(value string, t reflect.Type) any { if parsed, err := strconv.ParseFloat(value, 64); err == nil { return parsed } + case reflect.Slice: + // A list's default is written in the tag the way it reads in prose, + // `success, failure`, because that is what the field reference prints. + // The schema needs the value itself: a string default on an array + // property is a contradiction, and an editor that applies defaults + // would fill the list with one sentence. + parts := strings.Split(value, ",") + out := make([]any, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + out = append(out, schemaTagValue(part, target.Elem())) + } + return out } return value } diff --git a/internal/app/jsonschema_test.go b/internal/app/jsonschema_test.go index 34f3c76..3c74dd6 100644 --- a/internal/app/jsonschema_test.go +++ b/internal/app/jsonschema_test.go @@ -3,6 +3,7 @@ package app import ( "bytes" "encoding/json" + "fmt" "os" "path/filepath" "reflect" @@ -309,3 +310,62 @@ func TestEveryJSONSchemaExemptionNamesARealCase(t *testing.T) { } } } + +// A JSON Schema default has to be a value of the property's own type. A list +// whose default is a sentence is a contradiction, and an editor that applies +// defaults would fill the list with that sentence. +func TestPublishedSchemaGivesListFieldsListDefaultsAndExamples(t *testing.T) { + body, err := JSONSchema() + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatal(err) + } + var walk func(node map[string]any, path string) + walk = func(node map[string]any, path string) { + if node["type"] == "array" { + if value, ok := node["default"]; ok { + if _, isList := value.([]any); !isList { + t.Errorf("%s is an array whose default is %T (%v)", path, value, value) + } + } + for i, example := range asList(node["examples"]) { + if _, isList := example.([]any); !isList { + t.Errorf("%s example %d is %T (%v), not a value of the array type", path, i, example, example) + } + } + } + for _, key := range []string{"properties", "patternProperties"} { + for name, child := range asMap(node[key]) { + if sub, ok := child.(map[string]any); ok { + walk(sub, path+"."+name) + } + } + } + for _, key := range []string{"items", "additionalProperties"} { + if sub, ok := node[key].(map[string]any); ok { + walk(sub, path+"."+key) + } + } + for _, key := range []string{"allOf", "anyOf", "oneOf"} { + for i, branch := range asList(node[key]) { + if sub, ok := branch.(map[string]any); ok { + walk(sub, fmt.Sprintf("%s.%s[%d]", path, key, i)) + } + } + } + } + walk(doc, "") +} + +func asList(v any) []any { + out, _ := v.([]any) + return out +} + +func asMap(v any) map[string]any { + out, _ := v.(map[string]any) + return out +} diff --git a/site/public/onebox.run-v1.schema.json b/site/public/onebox.run-v1.schema.json index 76ba0e3..6c76f12 100644 --- a/site/public/onebox.run-v1.schema.json +++ b/site/public/onebox.run-v1.schema.json @@ -731,7 +731,9 @@ "backup_key_material": { "description": "Key-material identities the backup report must name.", "examples": [ - "BACKUP_ACCESS_KEY_ID" + [ + "BACKUP_ACCESS_KEY_ID" + ] ], "items": { "type": "string" @@ -1123,7 +1125,10 @@ "type": "string" }, "on": { - "default": "success, failure", + "default": [ + "success", + "failure" + ], "description": "Operation outcomes that trigger this notification.", "items": { "enum": [ @@ -2266,7 +2271,9 @@ "enum": { "description": "Accepted values.", "examples": [ - "catalog" + [ + "catalog" + ] ], "items": { "type": "string" @@ -2598,7 +2605,10 @@ "type": "string" }, "notify": { - "default": "failure, timeout", + "default": [ + "failure", + "timeout" + ], "description": "Run outcomes that send the configured notifications: success, failure, timeout, skipped.", "items": { "enum": [ From aad9230215400cd1718097b1e1d333a449f43a75 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 6 Sep 2026 07:37:09 -0700 Subject: [PATCH 24/25] fix(schedule): name the stand-aside note the same on both sides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner wrote `$state.skip.${INVOCATION_ID:-$$}` and the notifier looked for `$state.skip.${INVOCATION_ID:-}`. Under systemd the variable is always set, so the two agreed; anywhere else they did not, and a note the notifier cannot see sends it back to the state file belonging to the run in flight — the clobbering this note exists to prevent. Both sides now use one expression, and a test holds them to it. The comment on requireScheduleHost still described the systemd 252 floor as applying to every scheduled job, which it stopped doing when the floor was scoped back to jobs declaring inputs, and contradicted the comment inside the function. Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule.go | 12 +++++++----- internal/engine/schedule_test.go | 24 +++++++++++++++++++++++- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index f313974..39e7d17 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -281,7 +281,10 @@ func scheduleLockLines(names app.Names, job, applicationLock string, lockTTL tim // rather than the state a different run is still writing. Without the // note the notifier would see a clean exit and record this activation // as a success that never ran. - "skip_marker=\"$state.skip.${INVOCATION_ID:-$$}\"", + // Named for the activation, the same way the notifier looks for it. + // The two have to agree exactly: a marker the notifier cannot find + // sends it back to the state file, which is the run in flight's. + "skip_marker=\"$state.skip.${INVOCATION_ID:-}\"", "stand_aside() { umask 077; printf 'skipped=%s\\noperation=%s\\ninputs=%s\\n' \"$1\" \"$operation\" \"$inputs_json\" >\"$skip_marker\"; echo \"onebox: skipped: $1\" >&2; exit 0; }", "exec 9>" + q(names.ScheduledJobRunLock(job)), "/usr/bin/flock --exclusive --nonblock 9 || stand_aside 'another run of this job is still in progress'", @@ -296,10 +299,9 @@ func scheduleLockLines(names app.Names, job, applicationLock string, lockTTL tim // SyncSchedules asks again so `ob schedule apply` cannot bypass it. // // systemd 252 introduced TRIGGER_UNIT, which is how the runner tells a timer -// firing from an operator's start. On an older systemd every activation would -// look manual: recorded as such, and consuming a pending inputs file that was -// meant for the operator's run. The floor applies to every scheduled job, not -// only those with inputs, because the record's trigger is part of the contract. +// firing from an operator's start. The floor applies only to a job that +// declares inputs, and to `ob schedule run`; see below for why, and why a host +// that has been running scheduled jobs for years is not refused one. func (e *Engine) requireScheduleHost(ctx context.Context, jobs []app.ScheduledJob) error { if len(jobs) == 0 { return nil diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index bf68036..82be9a1 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "runtime" "strings" "testing" @@ -1447,7 +1448,7 @@ func TestScheduledJobRunnerDoesNotClobberARunningJobsState(t *testing.T) { runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) // The note is keyed to this activation, so it cannot be mistaken for the // state file of the run that holds the lock. - if !strings.Contains(runner, `skip_marker="$state.skip.${INVOCATION_ID:-$$}"`) || + if !strings.Contains(runner, `skip_marker="$state.skip.${INVOCATION_ID:-}"`) || !strings.Contains(runner, `stand_aside() { umask 077; printf 'skipped=%s\noperation=%s\ninputs=%s\n' "$1" "$operation" "$inputs_json" >"$skip_marker"`) { t.Fatalf("runner has no lock-less skip note:\n%s", runner) } @@ -1582,3 +1583,24 @@ func TestScheduledJobNotifierReadsAStandAsideNoteAndSpareTheRunningState(t *test t.Fatal("the stand-aside note survived its own notifier") } } + +// The runner writes the stand-aside note and the notifier reads it. They name +// it with the same expression or the note is invisible, and the notifier goes +// back to the state file belonging to the run that is still going. +func TestScheduleSkipMarkerIsNamedIdenticallyOnBothSides(t *testing.T) { + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + job := app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + cfg := testConfig() + e := New(cfg, testProject(t), &transport.Fake{TargetName: "root@example.internal"}, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + notifier, err := e.scheduleNotifier(job) + if err != nil { + t.Fatal(err) + } + marker := regexp.MustCompile(`skip_marker="[^"]+"`) + inRunner := marker.FindString(runner) + inNotifier := marker.FindString(notifier) + if inRunner == "" || inRunner != inNotifier { + t.Fatalf("runner names the note %q and the notifier %q", inRunner, inNotifier) + } +} From 342820fdbc557baf2ade29f99e1c50cded6aa128 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 6 Sep 2026 07:55:59 -0700 Subject: [PATCH 25/25] fix(schedule): report a manual run's own failure, and stop guessing about the host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five from the final review, all in the operator-initiated path. The journal said `schedule run ... started` whether or not the unit ever started. The finish is now written after the attempt and carries its real status, so `ob audit` can show a request that failed — a unit that does not exist because `ob schedule apply` was never run, or a `--wait` that saw the job fail. `ExecuteRequest.Validate` rejected every other kind's fields on the wrong kind but not `job`, `inputs` and `wait`, so a deploy carrying them was accepted and they were dropped in silence. `--wait` gave journald 1.8s to make the record queryable and then declared the run had never happened. It waits ten seconds and says what it actually knows: no record appeared yet, and where to look. A stand-aside note is removed only by the activation that wrote it, so one orphaned by a power cut stayed forever. Swept a day later, under the job lock. `hasTriggerUnit` cached a transport error as "this host is too old", which preflight then turned into a refused deploy. A failed probe is not an answer. Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5 --- internal/engine/schedule.go | 10 +++++- internal/engine/schedule_run.go | 38 ++++++++++++++------ internal/engine/schedule_run_test.go | 52 ++++++++++++++++++++++++++-- internal/onebox/execution_types.go | 3 ++ 4 files changed, 89 insertions(+), 14 deletions(-) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 39e7d17..3421b97 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -288,6 +288,11 @@ func scheduleLockLines(names app.Names, job, applicationLock string, lockTTL tim "stand_aside() { umask 077; printf 'skipped=%s\\noperation=%s\\ninputs=%s\\n' \"$1\" \"$operation\" \"$inputs_json\" >\"$skip_marker\"; echo \"onebox: skipped: $1\" >&2; exit 0; }", "exec 9>" + q(names.ScheduledJobRunLock(job)), "/usr/bin/flock --exclusive --nonblock 9 || stand_aside 'another run of this job is still in progress'", + // Only the activation that wrote a note removes it, so one lost + // between the runner exiting and ExecStopPost — a power cut, a killed + // systemd — would sit here forever. Swept a day later, under the job + // lock, which is long past any live note's few milliseconds. + "find " + q(names.AppDir()+"/schedule") + " -maxdepth 1 -name " + q(job+".state.skip.*") + " -mtime +1 -delete 2>/dev/null || true", "exec 8>" + q(names.ScheduleRunLock()), "/usr/bin/flock --exclusive --nonblock 8 || skip 'an application operation is taking its lock'", "if [ -e " + q(applicationLock) + " ] && [ \"$(" + lockAgeCmd(applicationLock) + ")\" -le " + strconv.Itoa(ttlSeconds) + " ]; then skip 'an application operation holds the deploy lock'; fi", @@ -328,10 +333,13 @@ func (e *Engine) hasTriggerUnit(ctx context.Context) bool { return e.triggerUnitPresent } res, err := e.T.Run(ctx, "systemctl --version 2>/dev/null | head -1") - e.triggerUnitProbed = true if err != nil { + // A transport failure says nothing about the host's systemd. Caching + // it would turn one flaky round trip into "this host is too old" for + // the rest of the operation, and preflight would refuse the deploy. return false } + e.triggerUnitProbed = true version, ok := systemdVersion(res.Stdout) e.triggerUnitPresent = ok && version >= 252 return e.triggerUnitPresent diff --git a/internal/engine/schedule_run.go b/internal/engine/schedule_run.go index eff62e8..4414b3b 100644 --- a/internal/engine/schedule_run.go +++ b/internal/engine/schedule_run.go @@ -33,7 +33,7 @@ type ScheduleRunResult struct { // runs any scheduled job unattended, but an operator choosing the moment and // the inputs is the case the sealed plan of `ob job run` exists for, and a // migration or destructive job keeps that path. -func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inputs map[string]string, wait bool) (ScheduleRunResult, error) { +func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inputs map[string]string, wait bool) (_ ScheduleRunResult, err error) { result := ScheduleRunResult{Job: name, Operation: operationID, Inputs: inputs} if strings.TrimSpace(operationID) == "" { return result, errors.New("schedule run requires an operation id") @@ -126,13 +126,24 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu if err := writer.Append(ctx, record); err != nil { return result, fmt.Errorf("journal schedule run start: %w", err) } - // The finish is written now, under the lock, because the outcome does not - // belong to this operation: it is the run record on the host, joined to - // this journal entry by the operation id the inputs file carries. - record.Event, record.Detail = "finish", "unit started; outcome in ob schedule history "+name - if err := writer.Append(ctx, record); err != nil { - return result, fmt.Errorf("journal schedule run finish: %w", err) - } + // The finish records how this request ended, not how the run did: the run + // has its own record on the host, joined to this entry by the operation + // id. But a request that never started the unit, or that waited and saw + // the job fail, is not a success, and `ob audit` has to be able to say so. + // + // Written after the lock is released, which is safe because a journal is + // per operation id: no other operation appends to this file. + defer func() { + finish := record + finish.Event, finish.Status = "finish", "ok" + finish.Detail = "unit started; outcome in ob schedule history " + name + if err != nil { + finish.Status, finish.Detail = "fail", err.Error() + } + if appendErr := writer.Append(ctx, finish); appendErr != nil { + err = errors.Join(err, fmt.Errorf("journal schedule run finish: %w", appendErr)) + } + }() e.ReleaseLock(ctx) locked = false @@ -190,10 +201,14 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu // blocking start returns, so a few short retries stand between the start and // the read. Matching on the operation id means a record left by an earlier // run, or by a timer firing that took this slot, is never reported as ours. +// The window is generous because journald ingests the notifier's line +// asynchronously: the blocking start has returned, so ExecStopPost has run, +// but the record may not be queryable yet on a loaded host. Giving up early +// would blame the host for a run that in fact succeeded. func (e *Engine) awaitScheduleRecord(ctx context.Context, name, operationID string) (*ScheduleRunRecord, error) { - for attempt := range 10 { + for attempt := range 40 { if attempt > 0 { - e.Opts.Sleep(200 * time.Millisecond) + e.Opts.Sleep(250 * time.Millisecond) } records, err := e.ScheduleHistory(ctx, name, 5) if err != nil { @@ -205,7 +220,8 @@ func (e *Engine) awaitScheduleRecord(ctx context.Context, name, operationID stri } } } - return nil, fmt.Errorf("no run record carries operation %s for job %s: the unit did not run for this request; a timer firing may have taken the slot, or the host's notifier wrote nothing", operationID, name) + return nil, fmt.Errorf("no run record for operation %s appeared within %s: the run may still be settling in the host journal, a timer firing may have taken the slot, or the notifier wrote nothing. ob schedule history %s shows what the host has", + operationID, 10*time.Second, name) } // discardInputs removes a pending inputs file this request wrote and can no diff --git a/internal/engine/schedule_run_test.go b/internal/engine/schedule_run_test.go index 863f866..9a7d0ae 100644 --- a/internal/engine/schedule_run_test.go +++ b/internal/engine/schedule_run_test.go @@ -61,8 +61,14 @@ func TestScheduleRunWritesInputsJournalsThenStartsAfterReleasingTheLock(t *testi t.Fatalf("journal is missing %q:\n%s", want, seq) } } - if journal := strings.Index(seq, `"phase":"schedule-run","event":"finish"`); journal > release { - t.Fatalf("journal finish was written after the lock was released:\n%s", seq) + // The start is claimed under the lock; the finish records how the request + // ended, so it comes after the unit was actually started. A journal is per + // operation id, so no other operation appends to this file meanwhile. + if started := strings.Index(seq, `"phase":"schedule-run","event":"start"`); started < 0 || started > release { + t.Fatalf("journal start was not written under the lock:\n%s", seq) + } + if finish := strings.LastIndex(seq, `"phase":"schedule-run","event":"finish","status":"ok"`); finish < start { + t.Fatalf("journal finish was written before the unit was started:\n%s", seq) } } @@ -259,3 +265,45 @@ func TestScheduleRunRefusesAHostThatCannotTellTheTriggerApart(t *testing.T) { } } } + +// The journal has to be able to say the request failed. Writing the finish +// as ok before the unit is even started left ob audit reporting "started" for +// a run that never began. +func TestScheduleRunJournalsAFailedRequestAsFailed(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl --version"): + return transport.Result{Stdout: "systemd 255 (255.4-1ubuntu8)\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "systemctl start"): + return transport.Result{ExitCode: 5, Stderr: "Unit ob-sample-sync.service not found."}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + if _, err := e.ScheduleRun(context.Background(), "op-1", "sync", nil, false); err == nil { + t.Fatal("a failed start was reported as success") + } + seq := strings.Join(f.Commands, "\n") + if !strings.Contains(seq, `"phase":"schedule-run","event":"finish","status":"fail"`) { + t.Fatalf("the journal calls a failed request started:\n%s", seq) + } + // The journal redacts a failure's detail on purpose, so the record says + // that the request failed and where to look, not what the host said. + if !strings.Contains(seq, `"error_code":"execution_failed"`) { + t.Fatalf("the failed finish carries no error code:\n%s", seq) + } + if strings.Contains(seq, `"phase":"schedule-run","event":"finish","status":"ok"`) { + t.Fatalf("a failed request also journaled a success:\n%s", seq) + } +} diff --git a/internal/onebox/execution_types.go b/internal/onebox/execution_types.go index 414f8b7..deb2c35 100644 --- a/internal/onebox/execution_types.go +++ b/internal/onebox/execution_types.go @@ -357,6 +357,9 @@ func (request ExecuteRequest) Validate() error { if (request.Approval != nil || request.BackupReport != nil || request.MigrationBackupOverride != nil) && request.Kind != KindDeploy && request.Kind != KindJobRun { return errors.New("approval and migration backup authorization are valid only for deploy and job run") } + if (request.Job != "" || len(request.Inputs) > 0 || request.Wait) && request.Kind != KindScheduleRun { + return errors.New("job, inputs and wait are valid only for schedule run") + } return nil }