diff --git a/cmd/ob-docgen/main.go b/cmd/ob-docgen/main.go index 118ffc00..9ad4bf5e 100644 --- a/cmd/ob-docgen/main.go +++ b/cmd/ob-docgen/main.go @@ -636,6 +636,17 @@ func defaultOf(node map[string]any) string { if !ok { return "" } + // A list default is an array in the schema, because that is the only + // thing a JSON Schema default for an array property may be. Printing it + // with Go's slice syntax puts `[failure timeout]` in a reference table + // nobody writes that way; the table reads as the value would be written. + if items, ok := v.([]any); ok { + parts := make([]string, 0, len(items)) + for _, item := range items { + parts = append(parts, fmt.Sprint(item)) + } + return strings.Join(parts, ", ") + } return fmt.Sprint(v) } diff --git a/cmd/ob/ops.go b/cmd/ob/ops.go index bc680605..1a2b28a7 100644 --- a/cmd/ob/ops.go +++ b/cmd/ob/ops.go @@ -64,28 +64,8 @@ func addOpsCommands(root *cobra.Command, g *globalFlags) { proxyCmd.AddCommand(proxyApplyCmd) root.AddCommand(proxyCmd) - // schedule apply — explicitly reconcile runner-owned host timers after an - // Onebox upgrade, without coupling package installation to remote mutation. - scheduleCmd := &cobra.Command{Use: "schedule", Short: "manage host timers for scheduled jobs", - Long: "Manage the systemd timers generated for scheduled jobs.\n\n" + - "Timers outlive the Onebox process and the package installed on the operator\n" + - "workstation. `apply` explicitly reconciles their units after a runner or\n" + - "configuration change without deploying a release.", - Args: cobra.NoArgs, RunE: showCommandHelp} - var scheduleBreakLock bool - scheduleApplyCmd := &cobra.Command{ - Use: "apply", - Short: "reconcile scheduled-job units without deploying a release", - Long: "Converge every declared scheduled-job timer, service, runner, and failure notifier to what the current Onebox runner generates.\n\nTaken under the application lock and fence so a deploy or host-fired job cannot modify the same runtime concurrently. This is the explicit post-upgrade path; upgrading the local package never mutates a remote host by itself.", - RunE: func(cmd *cobra.Command, _ []string) error { - return runMutation(cmd, g, onebox.ExecuteRequest{ - Kind: onebox.KindScheduleApply, BreakLock: scheduleBreakLock, - }, "schedule apply") - }, - } - scheduleApplyCmd.Flags().BoolVar(&scheduleBreakLock, "break-lock", false, "break a stale operation lock after inspecting its holder") - scheduleCmd.AddCommand(scheduleApplyCmd) - root.AddCommand(scheduleCmd) + // schedule apply | list | history | logs — see schedule.go. + addScheduleCommands(root, g) // secrets list | edit | push secretsCmd := &cobra.Command{Use: "secrets", Short: "SOPS-encrypted secrets", diff --git a/cmd/ob/output.go b/cmd/ob/output.go index e6f4cd33..86810774 100644 --- a/cmd/ob/output.go +++ b/cmd/ob/output.go @@ -76,43 +76,47 @@ const ( // Cobra's native human output. Keeping this list closed makes adding a command // an explicit CLI-contract decision instead of silently inheriting behavior. var cliOutputMatrix = map[string]cliOutputClass{ - "ob abort": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob approve": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob audit": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob bootstrap": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob canonical": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob deploy": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob destroy": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob doctor": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob eject": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob exec": {Class: cliClassOperatorPassthrough, NDJSON: true}, - "ob init": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob backup create": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup enable": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup disable": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup drill": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup restore": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup prune": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup verify": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob backup status": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob job plan": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob job run": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob logs": {Class: cliClassOperatorPassthrough, JSON: true, NDJSON: true}, - "ob plan": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob preflight": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob preview": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob proxy apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob resume": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob rollback": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob schedule apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob schema": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob secrets edit": {Class: cliClassTrustedEditor, JSON: true}, - "ob secrets list": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob secrets push": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob service apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob status": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob validate": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob version": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob abort": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob approve": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob audit": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob bootstrap": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob canonical": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob deploy": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob destroy": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob doctor": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob eject": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob exec": {Class: cliClassOperatorPassthrough, NDJSON: true}, + "ob init": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob backup create": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup enable": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup disable": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup drill": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup restore": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup prune": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup verify": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob backup status": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob job plan": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob job run": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob logs": {Class: cliClassOperatorPassthrough, JSON: true, NDJSON: true}, + "ob plan": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob preflight": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob preview": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob proxy apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob resume": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob rollback": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob schedule apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob schedule history": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob schedule list": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob schedule logs": {Class: cliClassOperatorPassthrough, JSON: true, NDJSON: true}, + "ob schedule run": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob schema": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob secrets edit": {Class: cliClassTrustedEditor, JSON: true}, + "ob secrets list": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob secrets push": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob service apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, + "ob status": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob validate": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob version": {Class: cliClassFiniteEnvelope, JSON: true}, } type cliExitError struct { diff --git a/cmd/ob/output_test.go b/cmd/ob/output_test.go index 3ded28d1..41e5a11e 100644 --- a/cmd/ob/output_test.go +++ b/cmd/ob/output_test.go @@ -490,43 +490,47 @@ workloads: func TestLeafOutputMatrixIsClosedAndHasNoAliases(t *testing.T) { wantMatrix := map[string]cliOutputClass{ - "ob abort": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob approve": {Class: "finite_envelope", JSON: true}, - "ob audit": {Class: "finite_envelope", JSON: true}, - "ob bootstrap": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob canonical": {Class: "finite_envelope", JSON: true}, - "ob deploy": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob destroy": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob doctor": {Class: "finite_envelope", JSON: true}, - "ob eject": {Class: "finite_envelope", JSON: true}, - "ob exec": {Class: "operator_passthrough", NDJSON: true}, - "ob init": {Class: "finite_envelope", JSON: true}, - "ob job plan": {Class: "finite_envelope", JSON: true}, - "ob backup create": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup enable": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup disable": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup drill": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup restore": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup prune": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup verify": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob backup status": {Class: "finite_envelope", JSON: true}, - "ob job run": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob logs": {Class: "operator_passthrough", JSON: true, NDJSON: true}, - "ob plan": {Class: "finite_envelope", JSON: true}, - "ob preflight": {Class: "finite_envelope", JSON: true}, - "ob preview": {Class: "finite_envelope", JSON: true}, - "ob proxy apply": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob resume": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob rollback": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob schedule apply": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob schema": {Class: "finite_envelope", JSON: true}, - "ob secrets edit": {Class: "trusted_editor", JSON: true}, - "ob secrets list": {Class: "finite_envelope", JSON: true}, - "ob secrets push": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob service apply": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob status": {Class: "finite_envelope", JSON: true}, - "ob validate": {Class: "finite_envelope", JSON: true}, - "ob version": {Class: "finite_envelope", JSON: true}, + "ob abort": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob approve": {Class: "finite_envelope", JSON: true}, + "ob audit": {Class: "finite_envelope", JSON: true}, + "ob bootstrap": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob canonical": {Class: "finite_envelope", JSON: true}, + "ob deploy": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob destroy": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob doctor": {Class: "finite_envelope", JSON: true}, + "ob eject": {Class: "finite_envelope", JSON: true}, + "ob exec": {Class: "operator_passthrough", NDJSON: true}, + "ob init": {Class: "finite_envelope", JSON: true}, + "ob job plan": {Class: "finite_envelope", JSON: true}, + "ob backup create": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup enable": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup disable": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup drill": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup restore": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup prune": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup verify": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob backup status": {Class: "finite_envelope", JSON: true}, + "ob job run": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob logs": {Class: "operator_passthrough", JSON: true, NDJSON: true}, + "ob plan": {Class: "finite_envelope", JSON: true}, + "ob preflight": {Class: "finite_envelope", JSON: true}, + "ob preview": {Class: "finite_envelope", JSON: true}, + "ob proxy apply": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob resume": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob rollback": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob schedule apply": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob schedule history": {Class: "finite_envelope", JSON: true}, + "ob schedule list": {Class: "finite_envelope", JSON: true}, + "ob schedule logs": {Class: "operator_passthrough", JSON: true, NDJSON: true}, + "ob schedule run": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob schema": {Class: "finite_envelope", JSON: true}, + "ob secrets edit": {Class: "trusted_editor", JSON: true}, + "ob secrets list": {Class: "finite_envelope", JSON: true}, + "ob secrets push": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob service apply": {Class: "finite_stream", JSON: true, NDJSON: true}, + "ob status": {Class: "finite_envelope", JSON: true}, + "ob validate": {Class: "finite_envelope", JSON: true}, + "ob version": {Class: "finite_envelope", JSON: true}, } if !reflect.DeepEqual(cliOutputMatrix, wantMatrix) { t.Fatalf("CLI output matrix changed:\ngot %#v\nwant %#v", cliOutputMatrix, wantMatrix) diff --git a/cmd/ob/schedule.go b/cmd/ob/schedule.go new file mode 100644 index 00000000..6549d9be --- /dev/null +++ b/cmd/ob/schedule.go @@ -0,0 +1,224 @@ +package main + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "text/tabwriter" + + "github.com/labstack/onebox/internal/onebox" + "github.com/spf13/cobra" +) + +// addScheduleCommands wires `ob schedule`: apply reconciles the host units, +// list, history and logs read what the host recorded. The read commands +// connect directly, like `ob status`; they hold no lock and write nothing. +func addScheduleCommands(root *cobra.Command, g *globalFlags) { + scheduleCmd := &cobra.Command{Use: "schedule", Short: "manage host timers for scheduled jobs", + Long: "Manage the systemd timers generated for scheduled jobs.\n\n" + + "Timers outlive the Onebox process and the package installed on the operator\n" + + "workstation. `apply` explicitly reconciles their units after a runner or\n" + + "configuration change without deploying a release. `list`, `history` and `logs`\n" + + "read the timer state and the run records the host keeps in its journal.", + Args: cobra.NoArgs, RunE: showCommandHelp} + + var scheduleBreakLock bool + scheduleApplyCmd := &cobra.Command{ + Use: "apply", + Short: "reconcile scheduled-job units without deploying a release", + Long: "Converge every declared scheduled-job timer, service, runner, and failure notifier to what the current Onebox runner generates.\n\nTaken under the application lock and fence so a deploy or host-fired job cannot modify the same runtime concurrently. This is the explicit post-upgrade path; upgrading the local package never mutates a remote host by itself.", + RunE: func(cmd *cobra.Command, _ []string) error { + return runMutation(cmd, g, onebox.ExecuteRequest{ + Kind: onebox.KindScheduleApply, BreakLock: scheduleBreakLock, + }, "schedule apply") + }, + } + scheduleApplyCmd.Flags().BoolVar(&scheduleBreakLock, "break-lock", false, "break a stale operation lock after inspecting its holder") + scheduleCmd.AddCommand(scheduleApplyCmd) + + listCmd := &cobra.Command{ + Use: "list", + Short: "declared scheduled jobs with timer state and next elapse", + Long: "List every job that declares a schedule beside what the host's timer says: whether it is active, when it fires next, and when it last fired. Reads only.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, p, err := loadAllLenient(cmd.Context(), g) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + defer cleanup() + jobs, err := e.ScheduleList(cmd.Context()) + if err != nil { + return writeStructuredCommandFailure(cmd, g, "schedule_list_failed", "scheduled jobs could not be listed", err) + } + if isStructuredOutput(g) { + return writeFiniteSuccess(cmd, g, map[string]any{"jobs": jobs}) + } + if len(jobs) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "no scheduled jobs declared") + return nil + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "JOB\tCRON\tTZ\tTIMER\tNEXT\tLAST TRIGGER\tPOLICY\tTIMEOUT") + for _, j := range jobs { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + j.Name, j.Cron, j.Timezone, orDash(j.TimerState), orDash(j.NextRun), orDash(j.LastTrigger), j.DeployLock, j.Timeout) + } + return w.Flush() + }, + } + scheduleCmd.AddCommand(listCmd) + + var historyCount int + historyCmd := &cobra.Command{ + Use: "history ", + Short: "run records of one scheduled job, newest first", + Long: "Read the run records the host wrote for one scheduled job. Each record is one activation: run id, trigger, release, start and end, attempts, exit status, outcome and, for a manual run, its inputs.\n\nRecords live in the host journal with syslog identifier ob-run and the job's unit in their ONEBOX_UNIT field; retention is the journal's. Reads only.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, p, err := loadAllLenient(cmd.Context(), g) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + defer cleanup() + records, err := e.ScheduleHistory(cmd.Context(), args[0], historyCount) + if err != nil { + return writeStructuredCommandFailure(cmd, g, "schedule_history_failed", "run history could not be read", err) + } + if isStructuredOutput(g) { + return writeFiniteSuccess(cmd, g, map[string]any{"job": args[0], "runs": records}) + } + if len(records) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "no recorded runs for %s\n", args[0]) + return nil + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "STARTED\tOUTCOME\tDURATION\tATTEMPTS\tEXIT\tTRIGGER\tRELEASE\tRUN") + for _, r := range records { + exit := "-" + if r.ExitStatus != nil { + exit = strconv.Itoa(*r.ExitStatus) + } + fmt.Fprintf(w, "%s\t%s\t%ds\t%d\t%s\t%s\t%s\t%s\n", + r.StartedAt, r.Outcome, r.DurationSeconds, r.Attempts, exit, r.Trigger, orDash(r.Release), r.Run) + } + return w.Flush() + }, + } + historyCmd.Flags().IntVarP(&historyCount, "count", "n", 20, "number of newest runs to show") + scheduleCmd.AddCommand(historyCmd) + + var logsRun string + logsCmd := &cobra.Command{ + Use: "logs ", + Short: "journal of one scheduled run", + Long: "Stream the host journal for one run of a scheduled job: by default the newest recorded run, or the run named with --run. The run id is systemd's invocation id, so the output is exactly that activation. Reads only.\n\nLog bytes are operator-controlled and may contain secrets; Onebox does not claim\nto redact passthrough output.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, p, err := loadAllLenient(cmd.Context(), g) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + defer cleanup() + if g.Output == "json" { + var stdout, stderr bytes.Buffer + run, err := e.ScheduleLogs(cmd.Context(), args[0], logsRun, &stdout, &stderr) + data := map[string]any{ + "job": args[0], "run": run, "stdout": stdout.String(), "stderr": stderr.String(), + "passthrough_unredacted": true, + } + if err != nil { + publicErr := publicError(err, "schedule_logs_failed", "run logs could not be read") + publicErr.Details = data + if writeErr := writeFiniteOutcome(cmd, g, cliOutcomeError, nil, publicErr); writeErr != nil { + return writeErr + } + return withExitCode(err, 1) + } + return writeFiniteSuccess(cmd, g, data) + } + if g.Output == "ndjson" { + stream := newCLIRecordStream(cmd.OutOrStdout(), commandName(cmd)) + run, err := e.ScheduleLogs(cmd.Context(), args[0], logsRun, stream.channelWriter("stdout"), stream.channelWriter("stderr")) + data := map[string]any{"job": args[0], "run": run, "passthrough_unredacted": true} + if err != nil { + if writeErr := stream.terminal(cliOutcomeError, nil, publicError(err, "schedule_logs_failed", "run logs could not be read")); writeErr != nil { + return writeErr + } + return withExitCode(err, 1) + } + return stream.terminal(cliOutcomeSuccess, data, nil) + } + if _, err := e.ScheduleLogs(cmd.Context(), args[0], logsRun, cmd.OutOrStdout(), cmd.ErrOrStderr()); err != nil { + return writeStructuredCommandFailure(cmd, g, "schedule_logs_failed", "run logs could not be read", err) + } + return nil + }, + } + logsCmd.Flags().StringVar(&logsRun, "run", "", "run id from ob schedule history; default the newest run") + scheduleCmd.AddCommand(logsCmd) + + var runInputs []string + var runWait, runBreakLock bool + runCmd := &cobra.Command{ + Use: "run ", + Short: "start a scheduled job now with declared inputs", + Long: "Start one scheduled job's unit now, with values for its declared inputs. Values are validated on the workstation against the declaration; an undeclared name or a value outside its enum or pattern is refused before anything reaches the host.\n\n" + + "Only a job with data_effect none may run this way; a migration or destructive job keeps the sealed plan of ob job run. The request is journaled as schedule_run with the operator and inputs, and the host record carries the operation id, so ob audit and ob schedule history join on it.\n\n" + + "The outcome is the run record: ob schedule history , or --wait to block until the unit exits and print it.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + inputs, err := parseScheduleInputs(runInputs) + if err != nil { + return writeEarlyOperationFailure(cmd, g, codedError("schedule_input_invalid", "%v", err)) + } + return runMutation(cmd, g, onebox.ExecuteRequest{ + Kind: onebox.KindScheduleRun, Job: args[0], Inputs: inputs, Wait: runWait, BreakLock: runBreakLock, + }, "schedule run") + }, + } + runCmd.Flags().StringArrayVar(&runInputs, "input", nil, "input override as NAME=VALUE; repeatable") + runCmd.Flags().BoolVar(&runWait, "wait", false, "block until the unit exits and report the run record") + runCmd.Flags().BoolVar(&runBreakLock, "break-lock", false, "break a stale operation lock after inspecting its holder") + scheduleCmd.AddCommand(runCmd) + + root.AddCommand(scheduleCmd) +} + +// parseScheduleInputs turns repeated --input NAME=VALUE flags into overrides. +// Only the shape is checked here; names and values are validated against the +// job's declaration by the engine before anything reaches the host. +func parseScheduleInputs(raw []string) (map[string]string, error) { + out := map[string]string{} + for _, item := range raw { + name, value, ok := strings.Cut(item, "=") + if !ok || name == "" { + return nil, fmt.Errorf("--input %q must be NAME=VALUE", item) + } + if _, dup := out[name]; dup { + return nil, fmt.Errorf("--input %s given twice", name) + } + out[name] = value + } + return out, nil +} + +func orDash(s string) string { + if strings.TrimSpace(s) == "" { + return "-" + } + return s +} diff --git a/cmd/ob/schedule_test.go b/cmd/ob/schedule_test.go new file mode 100644 index 00000000..4076d5fc --- /dev/null +++ b/cmd/ob/schedule_test.go @@ -0,0 +1,18 @@ +package main + +import "testing" + +func TestParseScheduleInputsFlags(t *testing.T) { + got, err := parseScheduleInputs([]string{"SOURCE=prices", "SINCE=2026-09-01=ish"}) + if err != nil || got["SOURCE"] != "prices" || got["SINCE"] != "2026-09-01=ish" { + t.Fatalf("got %#v, %v", got, err) + } + if got, err := parseScheduleInputs(nil); err != nil || len(got) != 0 { + t.Fatalf("no flags should mean no overrides: %#v, %v", got, err) + } + for _, bad := range [][]string{{"SOURCE"}, {"=x"}, {"SOURCE=a", "SOURCE=b"}} { + if _, err := parseScheduleInputs(bad); err == nil { + t.Errorf("%v was accepted", bad) + } + } +} diff --git a/docs/onebox.run-v1.schema.json b/docs/onebox.run-v1.schema.json index ef540909..6c76f12e 100644 --- a/docs/onebox.run-v1.schema.json +++ b/docs/onebox.run-v1.schema.json @@ -731,7 +731,9 @@ "backup_key_material": { "description": "Key-material identities the backup report must name.", "examples": [ - "BACKUP_ACCESS_KEY_ID" + [ + "BACKUP_ACCESS_KEY_ID" + ] ], "items": { "type": "string" @@ -1123,7 +1125,10 @@ "type": "string" }, "on": { - "default": "success, failure", + "default": [ + "success", + "failure" + ], "description": "Operation outcomes that trigger this notification.", "items": { "enum": [ @@ -1878,6 +1883,11 @@ "required": [ "schedule" ] + }, + { + "required": [ + "inputs" + ] } ] } @@ -2231,6 +2241,64 @@ "description": "Run a minimal init process as PID 1 inside the container.", "type": "boolean" }, + "inputs": { + "additionalProperties": { + "additionalProperties": false, + "oneOf": [ + { + "required": [ + "enum" + ] + }, + { + "required": [ + "pattern" + ] + } + ], + "patternProperties": { + "^x-": {} + }, + "properties": { + "default": { + "description": "Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint.", + "type": "string" + }, + "description": { + "description": "What the input controls.", + "type": "string" + }, + "enum": { + "description": "Accepted values.", + "examples": [ + [ + "catalog" + ] + ], + "items": { + "type": "string" + }, + "type": "array" + }, + "pattern": { + "description": "Regular expression the whole value must match.", + "examples": [ + "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + ], + "type": "string" + } + }, + "required": [ + "default" + ], + "type": "object" + }, + "description": "Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them.", + "propertyNames": { + "pattern": "^[A-Z][A-Z0-9_]*$" + }, + "type": "object" + }, "labels": { "additionalProperties": {}, "description": "Additional container labels outside namespaces reserved by Onebox and the proxy.", @@ -2536,6 +2604,61 @@ ], "type": "string" }, + "notify": { + "default": [ + "failure", + "timeout" + ], + "description": "Run outcomes that send the configured notifications: success, failure, timeout, skipped.", + "items": { + "enum": [ + "success", + "failure", + "timeout", + "skipped" + ], + "type": "string" + }, + "type": "array" + }, + "retry": { + "additionalProperties": false, + "description": "Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run.", + "patternProperties": { + "^x-": {} + }, + "properties": { + "attempts": { + "default": 1, + "description": "Total attempts including the first, 1 to 10.", + "examples": [ + 3 + ], + "maximum": 10, + "minimum": 1, + "type": "integer" + }, + "backoff": { + "default": "30s", + "description": "Sleep before the second attempt; it doubles after each failure. Expects a duration such as 30s, 5m, 1h30m or 14d.", + "examples": [ + "1m" + ], + "pattern": "^(([0-9]+([.][0-9]+)?(ns|us|µs|ms|s|m|h))+|[0-9]+d)$", + "type": "string" + }, + "max_backoff": { + "default": "10m", + "description": "Upper bound for the doubling sleep. Expects a duration such as 30s, 5m, 1h30m or 14d.", + "examples": [ + "30m" + ], + "pattern": "^(([0-9]+([.][0-9]+)?(ns|us|µs|ms|s|m|h))+|[0-9]+d)$", + "type": "string" + } + }, + "type": "object" + }, "timeout": { "default": "1h", "description": "Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d.", diff --git a/docs/superpowers/plans/2026-09-05-scheduled-job-runs.md b/docs/superpowers/plans/2026-09-05-scheduled-job-runs.md new file mode 100644 index 00000000..99c3c883 --- /dev/null +++ b/docs/superpowers/plans/2026-09-05-scheduled-job-runs.md @@ -0,0 +1,2484 @@ +# Scheduled Job Runs Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give scheduled jobs a per-run record in the host journal, bounded retry with backoff, per-outcome notifications, and declared inputs with an operator-initiated `ob schedule run`, without any resident process or new store on the host. + +**Architecture:** The generated runner script gains a state file, a retry loop and manual-input handling; the generated `ExecStopPost` notifier finalises one JSON record per run into the journal via `systemd-cat -t ob-run` and sends notifications per outcome. `ob` reads records back with `journalctl` (`schedule history`, `schedule logs`, `schedule list`, richer `status`) and starts a unit with validated inputs (`schedule run`), journaled as `schedule_run`. Every new project-file field is optional and every default preserves today's behaviour. + +**Tech Stack:** Go 1.27, cobra CLI, POSIX sh rendered from Go, systemd (timers, `INVOCATION_ID`, `TRIGGER_UNIT`, `SuccessExitStatus`, `systemd-cat`, `journalctl`), `transport.Fake` for unit tests, Lima Ubuntu 24.04 for `just server-e2e`. + +**Spec:** https://github.com/labstack/onebox/issues/155 (revised body, 2026-09-05) + +## Global Constraints + +- Onebox environment variables use the `ONEBOX_` prefix. `OB_` is retired and `just env-namespace` fails on any tracked-file match of `\bOB_[A-Z0-9_]+`. Reserved input prefix and metadata line are therefore `ONEBOX_`. +- Workload names match `^[a-z]([a-z0-9-]{0,38}[a-z0-9])?$`. +- Input names match `^[A-Z][A-Z0-9_]*$`, must not start with `ONEBOX_`, must not equal a key in the workload's `env`. +- Input values (defaults and overrides): at most 256 bytes, no `"`, no `\`, no control characters (`0x00-0x1f`, `0x7f`). Patterns match the whole value. +- `retry.attempts` 1..10 default 1; `retry.backoff` default `30s`; `retry.max_backoff` default `10m`; worst-case total backoff must be strictly less than `schedule.timeout`. +- `schedule.notify` values: `success`, `failure`, `timeout`, `skipped`; default `[failure, timeout]`. +- `inputs` requires `schedule` and `data_effect: none`. `ob schedule run` refuses any other data effect. +- Service unit gains `SuccessExitStatus=75`. +- Run record syslog identifier: `ob-run`. Record fields: `run, job, trigger, operation, release, started_at, finished_at, duration_s, attempts, exit_status, outcome, inputs`. +- Manual trigger detection: `$TRIGGER_UNIT` unset. Host needs systemd 252+ only when a job declares `inputs`; checked in `SyncSchedules` beside the existing `systemd-analyze calendar` check. +- Commit messages end with `Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5`. +- Gates before every commit: `go build ./... && go vet ./... && go test ./...`. Before the final commit of each slice also `just docs-generate`, `go run ./cmd/ob schema --out docs/onebox.run-v1.schema.json`, `just docs-generate-check`, `just env-namespace`. + +--- + +## Slice 1: History + +### Task 1: State file in both runner modes, `SuccessExitStatus=75` + +**Files:** +- Modify: `internal/engine/schedule.go:172-245` (`scheduleRunnerScript`, `pinnedScheduleRunnerScript`), `internal/engine/schedule.go:262-280` (`scheduleServiceUnit`) +- Test: `internal/engine/schedule_test.go` + +**Interfaces:** +- Produces: shell function `write_state ` defined in both runners, writing `release=`, `started_at=`, `started_epoch=`, `trigger=`, `operation=`, `attempt=`, `inputs=` lines atomically to `names.ScheduledJobRunState(job)`. Runner no longer deletes the state file. Go helper `scheduleStateFunction(state string) []string`. + +- [ ] **Step 1: Write the failing test** + +Append to `internal/engine/schedule_test.go`: + +```go +func TestScheduledJobRunnersRecordRunStateForTheNotifier(t *testing.T) { + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + for _, tc := range []struct { + name string + job app.ScheduledJob + }{ + {"exclusive", app.ScheduledJob{Name: "nightly", Cron: "0 2 * * *", Timezone: "UTC", Calendar: "*-*-* 02:00:00", Timeout: "45m", DeployLock: "exclusive"}}, + {"pinned", app.ScheduledJob{Name: "nightly", Cron: "0 2 * * *", Timezone: "UTC", Calendar: "*-*-* 02:00:00", Timeout: "45m", DeployLock: "pinned"}}, + } { + t.Run(tc.name, func(t *testing.T) { + runner := scheduleRunnerScript("sample", tc.job, names, "/var/lib/ob/sample/lock", nil) + for _, want := range []string{ + "state='/var/lib/ob/sample/schedule/nightly.state'", + "write_state() {", + "started_epoch=%s", + `trigger=%s`, + "write_state 1", + "mv -f \"$tmp\" \"$state\"", + } { + if !strings.Contains(runner, want) { + t.Errorf("%s runner is missing %q:\n%s", tc.name, want, runner) + } + } + if strings.Contains(runner, `rm -f "$state"`) { + t.Errorf("%s runner removes the state the notifier finalises:\n%s", tc.name, runner) + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(runner) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("%s runner is not valid POSIX shell: %v: %s\n%s", tc.name, err, output, runner) + } + }) + } + service := scheduleServiceUnit("sample", app.ScheduledJob{Name: "nightly", Timeout: "45m"}, + "/etc/systemd/system/ob-sample-nightly.run", "/etc/systemd/system/ob-sample-nightly.notify") + if !strings.Contains(service, "SuccessExitStatus=75") { + t.Errorf("a lock-conflict skip must not be a failed unit:\n%s", service) + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `go test ./internal/engine -run TestScheduledJobRunnersRecordRunStateForTheNotifier` +Expected: FAIL, runner is missing `write_state() {`. + +- [ ] **Step 3: Implement** + +In `internal/engine/schedule.go` add after `scheduleContainerCleanup`: + +```go +// scheduleStateFunction renders the shell function both runners use to record +// the run in progress. The notifier reads it after the run ends, so the runner +// never removes it: a runner that cleaned up its own state would erase the +// only evidence a timed-out run leaves behind. +func scheduleStateFunction() []string { + return []string{ + "write_state() {", + " umask 077", + " printf 'release=%s\\nstarted_at=%s\\nstarted_epoch=%s\\ntrigger=%s\\noperation=%s\\nattempt=%s\\ninputs=%s\\n' " + + "\"$release\" \"$started_at\" \"$started_epoch\" \"$trigger\" \"$operation\" \"$1\" \"$inputs_json\" >\"$tmp\"", + " mv -f \"$tmp\" \"$state\"", + "}", + } +} + +// scheduleRunPreamble sets the variables write_state records. The trigger is +// systemd's own word for it: a timer activation carries TRIGGER_UNIT (systemd +// 252+), anything else is an operator. +func scheduleRunPreamble(state string) []string { + return append([]string{ + "state=" + q(state), + "tmp=\"$state.$$\"", + "started_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", + "started_epoch=$(date -u '+%s')", + "if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=manual; fi", + "operation=''", + "inputs_json=''", + }, scheduleStateFunction()...) +} +``` + +Rewrite the exclusive runner body (replace lines from `scheduleContainerCleanup(container),` through `compose,` in `scheduleRunnerScript`): + +```go + lines := []string{ + "#!/bin/sh", + "# Written by Onebox. Edits are overwritten on the next deploy.", + "set -eu", + "install -d -m 700 " + q(names.AppDir()+"/schedule"), + "exec 9>" + q(names.ScheduledJobRunLock(job.Name)), + "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 9", + "exec 8>" + q(names.ScheduleRunLock()), + "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 8", + "if [ -e " + q(applicationLock) + " ]; then echo 'onebox: an application operation holds the deploy lock' >&2; exit 75; fi", + "release_dir=$(readlink -f " + q(names.CurrentLink()) + " 2>/dev/null || true)", + "release=${release_dir##*/}", + scheduleContainerCleanup(container), + "cleanup() { " + scheduleContainerCleanup(container) + "; rm -f \"$tmp\"; }", + "trap cleanup 0", + "trap 'exit 129' 1", + "trap 'exit 130' 2", + "trap 'exit 143' 15", + } + lines = append(lines, scheduleRunPreamble(names.ScheduledJobRunState(job.Name))...) + lines = append(lines, "write_state 1", compose, "") + return strings.Join(lines, "\n") +``` + +In `pinnedScheduleRunnerScript` replace the block from `"state=" + q(state),` through `"mv -f \"$tmp\" \"$state\"",` with: + +```go + "cleanup() { " + scheduleContainerCleanup(container) + "; rm -f \"$tmp\"; }", + "trap cleanup 0", + "trap 'exit 129' 1", + "trap 'exit 130' 2", + "trap 'exit 143' 15", + } + lines = append(lines, scheduleRunPreamble(state)...) + lines = append(lines, "write_state 1", "/usr/bin/flock --unlock 8", compose, "") + return strings.Join(lines, "\n") +``` + +Remove the now-unused `"started_at=$(date ...)"`, `"umask 077"`, `"printf 'release=...'"` lines and the old `"state="`/`"tmp="` lines from the pinned runner. Keep `"release=${release_dir##*/}"` in the pinned runner (it already exists). + +In `scheduleServiceUnit` add after `"TimeoutStartSec=" + job.Timeout,`: + +```go + // Exit 75 is the runner's "skipped for a lock conflict". It is a fact + // about timing, not a failure of the job, and it must not leave the + // unit failed or trip failure notifications. + "SuccessExitStatus=75", +``` + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine` +Expected: PASS. `TestScheduleStatusReportsRunningPinnedRelease` still passes because the status reader only needs `release=` and `started_at=` lines. + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine/schedule.go internal/engine/schedule_test.go +git commit -m "feat(schedule): record run state in both runner modes and stop treating a skip as failure + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 2: Notifier finalises one journal record per run + +**Files:** +- Modify: `internal/engine/schedule.go:287-335` (`scheduleFailureNotifier`) +- Test: `internal/engine/schedule_test.go` + +**Interfaces:** +- Produces: the notifier script reads the state file, derives `outcome`, writes one JSON line via `systemd-cat -t ob-run`, deletes the state file, then keeps today's failure/timeout notification behaviour. Constant `scheduleRunIdentifier = "ob-run"`. + +- [ ] **Step 1: Write the failing test** + +```go +func TestScheduledJobNotifierWritesOneRunRecordToTheJournal(t *testing.T) { + cfg := testConfig() + f := &transport.Fake{TargetName: "root@example.internal"} + e := New(cfg, testProject(t), f, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + script, err := e.scheduleFailureNotifier("nightly") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "state='/var/lib/ob/sample/schedule/nightly.state'", + `started_epoch=*) started_epoch=${line#started_epoch=}`, + `rm -f "$state"`, + `result=${SERVICE_RESULT:-success}`, + `status=${EXIT_STATUS:-0}`, + `outcome=timeout`, + `outcome=skipped`, + `outcome=success`, + `outcome=failure`, + `"run":"%s","job":"%s","trigger":"%s","operation":"%s","release":"%s"`, + `"duration_s":%s,"attempts":%s,"exit_status":%s,"outcome":"%s","inputs":{%s}`, + `"${INVOCATION_ID:-}" 'nightly'`, + "systemd-cat -t ob-run", + } { + if !strings.Contains(script, want) { + t.Errorf("notifier is missing %q:\n%s", want, script) + } + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(script) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("notifier is not valid POSIX shell: %v: %s\n%s", err, output, script) + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `go test ./internal/engine -run TestScheduledJobNotifierWritesOneRunRecordToTheJournal` +Expected: FAIL, missing `state='...'`. + +- [ ] **Step 3: Implement** + +Add the constant and a record-rendering helper near `scheduleNotificationTimestamp`: + +```go +// scheduleRunIdentifier is the syslog identifier of the one line the notifier +// writes per run. `journalctl -u -t ob-run` is the run history. +const scheduleRunIdentifier = "ob-run" + +// scheduleRunRecordLines finalises the run the runner started. It runs in +// ExecStopPost because only systemd knows how the run ended: a timed-out +// runner is killed mid-sleep and cannot write its own outcome. +func scheduleRunRecordLines(job, state string) []string { + return []string{ + "state=" + q(state), + "release=''; started_at=''; started_epoch=''; trigger=''; operation=''; attempt=0; inputs=''", + "if [ -f \"$state\" ]; then", + " while IFS= read -r line || [ -n \"$line\" ]; do", + " case \"$line\" in", + " release=*) release=${line#release=} ;;", + " started_at=*) started_at=${line#started_at=} ;;", + " started_epoch=*) started_epoch=${line#started_epoch=} ;;", + " trigger=*) trigger=${line#trigger=} ;;", + " operation=*) operation=${line#operation=} ;;", + " attempt=*) attempt=${line#attempt=} ;;", + " inputs=*) inputs=${line#inputs=} ;;", + " esac", + " done <\"$state\"", + " rm -f \"$state\"", + "fi", + "if [ -z \"$trigger\" ]; then if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=manual; fi; fi", + "result=${SERVICE_RESULT:-success}", + "status=${EXIT_STATUS:-0}", + "case \"$status\" in ''|*[!0-9]*) status=null ;; esac", + "case \"$attempt\" in ''|*[!0-9]*) attempt=0 ;; esac", + "if [ \"$result\" = timeout ]; then outcome=timeout", + "elif [ \"$status\" = 75 ]; then outcome=skipped", + "elif [ \"$result\" = success ] && [ \"$status\" = 0 ]; then outcome=success", + "else outcome=failure; fi", + "finished_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", + "now=$(date -u '+%s')", + "duration=0", + "case \"$started_epoch\" in ''|*[!0-9]*) ;; *) duration=$((now - started_epoch)) ;; esac", + "[ -z \"$started_at\" ] && started_at=$finished_at", + "printf '{\"run\":\"%s\",\"job\":\"%s\",\"trigger\":\"%s\",\"operation\":\"%s\",\"release\":\"%s\",\"started_at\":\"%s\",\"finished_at\":\"%s\",\"duration_s\":%s,\"attempts\":%s,\"exit_status\":%s,\"outcome\":\"%s\",\"inputs\":{%s}}\\n' " + + "\"${INVOCATION_ID:-}\" " + q(job) + " \"$trigger\" \"$operation\" \"$release\" \"$started_at\" \"$finished_at\" \"$duration\" \"$attempt\" \"$status\" \"$outcome\" \"$inputs\" " + + "| systemd-cat -t " + scheduleRunIdentifier + " || true", + } +} +``` + +In `scheduleFailureNotifier`, replace the line `` `[ "${SERVICE_RESULT:-success}" = success ] && exit 0`, `` with: + +```go + } + lines = append(lines, scheduleRunRecordLines(job, e.names().ScheduledJobRunState(job))...) + lines = append(lines, `case "$outcome" in failure|timeout) ;; *) exit 0 ;; esac`) + lines = append(lines, "") + if false { +``` + +Concretely the function becomes: + +```go + lines := []string{ + "#!/bin/sh", + "# Written by Onebox. Edits are overwritten on the next deploy.", + "set -u", + "exec 9>" + q(e.names().ScheduledJobRunLock(job)), + "if /usr/bin/flock --exclusive --nonblock 9; then", + " " + scheduleContainerCleanup(e.names().Container(job, 1)), + "fi", + } + lines = append(lines, scheduleRunRecordLines(job, e.names().ScheduledJobRunState(job))...) + lines = append(lines, `case "$outcome" in failure|timeout) ;; *) exit 0 ;; esac`) + var sends []string + // ... unchanged send rendering ... +``` + +Update `TestScheduledJobFailureNotifierUsesConfiguredWebhooks`: replace the expectation `` `${SERVICE_RESULT:-success}` `` with `` `result=${SERVICE_RESULT:-success}` ``. + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine/schedule.go internal/engine/schedule_test.go +git commit -m "feat(schedule): finalise one journal record per scheduled run from ExecStopPost + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 3: Read run records back: `ScheduleHistory`, `ScheduleList`, run logs + +**Files:** +- Create: `internal/engine/schedule_history.go` +- Test: `internal/engine/schedule_history_test.go` + +**Interfaces:** +- Produces: + - `type ScheduleRunRecord struct { Run, Job, Trigger, Operation, Release, StartedAt, FinishedAt string; DurationSeconds int; Attempts int; ExitStatus *int; Outcome string; Inputs map[string]string }` with JSON tags `run, job, trigger, operation, release, started_at, finished_at, duration_s, attempts, exit_status, outcome, inputs`. + - `func parseScheduleRunRecords(stdout string) []ScheduleRunRecord` (skips unparseable lines). + - `func scheduleHistoryCommand(unit string, n int) string` = `journalctl -u .service -t ob-run -o cat -r -n --no-pager 2>/dev/null || true`. + - `func (e *Engine) ScheduleHistory(ctx, job string, n int) ([]ScheduleRunRecord, error)`; unknown or unscheduled job returns an error `job %q is not a scheduled job`. + - `type ScheduleListing struct { Name, Unit, Cron, Timezone, DeployLock, Timeout, TimerState, NextRun, LastTrigger string }` JSON tags `name, unit, cron, timezone, deploy_lock, timeout, timer_state, next_run, last_trigger`. + - `func (e *Engine) ScheduleList(ctx) ([]ScheduleListing, error)` using `systemctl show .timer --no-pager --property=ActiveState --property=NextElapseUSecRealtime --property=LastTriggerUSec` batched with `@@` markers. + - `func (e *Engine) ScheduleLogs(ctx, job, run string, tail int, stdout, stderr io.Writer) error`: with `run` empty, resolve the newest record's run id via `ScheduleHistory(ctx, job, 1)`; command `journalctl _SYSTEMD_INVOCATION_ID= --no-pager -o short-iso`; with no record fall back to `journalctl -u .service -n --no-pager -o short-iso`. Run ids are validated against `^[0-9a-f]{32}$` before reaching a shell. + +- [ ] **Step 1: Write the failing tests** + +`internal/engine/schedule_history_test.go`: + +```go +package engine + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/transport" +) + +const sampleRunRecords = `{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"manual","operation":"20260905-151200-schedule_run-7c1e","release":"20260905-140000-ab12cd","started_at":"2026-09-05T15:12:01Z","finished_at":"2026-09-05T15:12:31Z","duration_s":30,"attempts":2,"exit_status":0,"outcome":"success","inputs":{"SOURCE":"prices"}} +not json +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","operation":"","release":"20260905-140000-ab12cd","started_at":"2026-09-05T15:00:01Z","finished_at":"2026-09-05T15:00:02Z","duration_s":1,"attempts":0,"exit_status":75,"outcome":"skipped","inputs":{}} +` + +func TestParseScheduleRunRecordsSkipsNoiseAndKeepsOrder(t *testing.T) { + records := parseScheduleRunRecords(sampleRunRecords) + if len(records) != 2 { + t.Fatalf("records = %#v", records) + } + if records[0].Run != "b2c3d4e5f60718293a4b5c6d7e8f9012" || records[0].Trigger != "manual" || records[0].Attempts != 2 || + records[0].DurationSeconds != 30 || records[0].Inputs["SOURCE"] != "prices" || records[0].ExitStatus == nil || *records[0].ExitStatus != 0 { + t.Fatalf("first record was not decoded: %#v", records[0]) + } + if records[1].Outcome != "skipped" || *records[1].ExitStatus != 75 { + t.Fatalf("second record was not decoded: %#v", records[1]) + } +} + +func scheduledFixture(t *testing.T) (*Engine, *transport.Fake) { + t.Helper() + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{} + return New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}), f +} + +func TestScheduleHistoryReadsTheUnitJournalNewestFirst(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "journalctl") { + return transport.Result{Stdout: sampleRunRecords}, true + } + return transport.Result{}, false + } + records, err := e.ScheduleHistory(context.Background(), "nightly", 20) + if err != nil { + t.Fatal(err) + } + if len(records) != 2 || records[0].Outcome != "success" { + t.Fatalf("records = %#v", records) + } + seq := strings.Join(f.Commands, "\n") + for _, want := range []string{"journalctl -u ob-sample-nightly.service", "-t ob-run", "-o cat", "-r", "-n 20", "--no-pager"} { + if !strings.Contains(seq, want) { + t.Fatalf("history read is missing %q:\n%s", want, seq) + } + } + if _, err := e.ScheduleHistory(context.Background(), "web", 20); err == nil { + t.Fatal("history of a non-scheduled workload was not refused") + } +} + +func TestScheduleListReadsTimerState(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: "@@nightly\nActiveState=active\nNextElapseUSecRealtime=Sat 2026-09-06 02:00:00 UTC\nLastTriggerUSec=Fri 2026-09-05 02:00:00 UTC\n"}, true + } + return transport.Result{}, false + } + listing, err := e.ScheduleList(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(listing) != 1 || listing[0].Unit != "ob-sample-nightly" || listing[0].TimerState != "active" || + listing[0].NextRun != "Sat 2026-09-06 02:00:00 UTC" || listing[0].LastTrigger != "Fri 2026-09-05 02:00:00 UTC" || listing[0].Cron != "0 2 * * *" { + t.Fatalf("listing = %#v", listing) + } +} + +func TestScheduleLogsTargetsOneInvocation(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "-t ob-run") { + return transport.Result{Stdout: sampleRunRecords}, true + } + return transport.Result{}, false + } + var out bytes.Buffer + if err := e.ScheduleLogs(context.Background(), "nightly", "", 200, &out, &out); err != nil { + t.Fatal(err) + } + seq := strings.Join(f.Commands, "\n") + if !strings.Contains(seq, "journalctl _SYSTEMD_INVOCATION_ID=b2c3d4e5f60718293a4b5c6d7e8f9012") { + t.Fatalf("logs did not target the newest run:\n%s", seq) + } + if err := e.ScheduleLogs(context.Background(), "nightly", "../etc", 200, &out, &out); err == nil { + t.Fatal("an invalid run id reached the shell") + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/engine -run 'TestParseScheduleRunRecords|TestScheduleHistory|TestScheduleList|TestScheduleLogs'` +Expected: FAIL to compile (undefined symbols). + +- [ ] **Step 3: Implement `internal/engine/schedule_history.go`** + +```go +package engine + +import ( + "context" + "encoding/json" + "fmt" + "io" + "regexp" + "strconv" + "strings" + + "github.com/labstack/onebox/internal/app" +) + +// ScheduleRunRecord is the line the notifier writes to the journal when a +// scheduled run ends. The journal is the store: there is no file to trim and +// nothing that can disagree with the unit's own log. +type ScheduleRunRecord struct { + Run string `json:"run"` + Job string `json:"job"` + Trigger string `json:"trigger"` + Operation string `json:"operation,omitempty"` + Release string `json:"release,omitempty"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + DurationSeconds int `json:"duration_s"` + Attempts int `json:"attempts"` + ExitStatus *int `json:"exit_status"` + Outcome string `json:"outcome"` + Inputs map[string]string `json:"inputs,omitempty"` +} + +// ScheduleListing is one declared job beside its timer as the host reports it. +type ScheduleListing struct { + Name string `json:"name"` + Unit string `json:"unit"` + Cron string `json:"cron"` + Timezone string `json:"timezone"` + DeployLock string `json:"deploy_lock"` + Timeout string `json:"timeout"` + TimerState string `json:"timer_state"` + NextRun string `json:"next_run,omitempty"` + LastTrigger string `json:"last_trigger,omitempty"` +} + +var scheduleRunID = regexp.MustCompile(`^[0-9a-f]{32}$`) + +func scheduleHistoryCommand(unit string, n int) string { + if n <= 0 { + n = 20 + } + return "journalctl -u " + q(unit+".service") + " -t " + scheduleRunIdentifier + + " -o cat -r -n " + strconv.Itoa(n) + " --no-pager 2>/dev/null || true" +} + +func parseScheduleRunRecords(stdout string) []ScheduleRunRecord { + var out []ScheduleRunRecord + for _, line := range strings.Split(stdout, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "{") { + continue + } + var record ScheduleRunRecord + if err := json.Unmarshal([]byte(line), &record); err != nil { + continue + } + out = append(out, record) + } + return out +} + +func (e *Engine) scheduledJob(name string) (app.ScheduledJob, error) { + jobs, err := e.Spec.ScheduledJobs() + if err != nil { + return app.ScheduledJob{}, err + } + for _, job := range jobs { + if job.Name == name { + return job, nil + } + } + return app.ScheduledJob{}, fmt.Errorf("job %q is not a scheduled job", name) +} + +// ScheduleHistory returns the newest n run records of one job, newest first. +func (e *Engine) ScheduleHistory(ctx context.Context, name string, n int) ([]ScheduleRunRecord, error) { + job, err := e.scheduledJob(name) + if err != nil { + return nil, err + } + res, err := e.T.Run(ctx, scheduleHistoryCommand(e.names().ScheduledJobUnit(job.Name), n)) + if err != nil { + return nil, err + } + records := parseScheduleRunRecords(res.Stdout) + if records == nil { + records = []ScheduleRunRecord{} + } + return records, nil +} + +// ScheduleList reads every declared job's timer in one round trip. +func (e *Engine) ScheduleList(ctx context.Context) ([]ScheduleListing, error) { + jobs, err := e.Spec.ScheduledJobs() + if err != nil { + return nil, err + } + if len(jobs) == 0 { + return []ScheduleListing{}, nil + } + var commands []string + for _, job := range jobs { + unit := e.names().ScheduledJobUnit(job.Name) + commands = append(commands, + "printf '%s\\n' "+q("@@"+job.Name), + "systemctl show "+q(unit+".timer")+" --no-pager --property=ActiveState --property=NextElapseUSecRealtime --property=LastTriggerUSec 2>/dev/null || true") + } + res, err := e.T.Run(ctx, strings.Join(commands, "\n")) + if err != nil { + return nil, err + } + observed := map[string]map[string]string{} + current := "" + for _, line := range strings.Split(res.Stdout, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "@@") { + current = strings.TrimPrefix(line, "@@") + observed[current] = map[string]string{} + continue + } + if key, value, ok := strings.Cut(line, "="); ok && current != "" { + observed[current][key] = value + } + } + out := make([]ScheduleListing, 0, len(jobs)) + for _, job := range jobs { + values := observed[job.Name] + out = append(out, ScheduleListing{ + Name: job.Name, Unit: e.names().ScheduledJobUnit(job.Name), Cron: job.Cron, Timezone: job.Timezone, + DeployLock: job.DeployLock, Timeout: job.Timeout, TimerState: values["ActiveState"], + NextRun: values["NextElapseUSecRealtime"], LastTrigger: values["LastTriggerUSec"], + }) + } + return out, nil +} + +// ScheduleLogs streams the journal of one run. The run id is systemd's +// invocation id, so the output is exactly that activation and nothing else. +func (e *Engine) ScheduleLogs(ctx context.Context, name, run string, tail int, stdout, stderr io.Writer) error { + job, err := e.scheduledJob(name) + if err != nil { + return err + } + unit := e.names().ScheduledJobUnit(job.Name) + if run == "" { + records, err := e.ScheduleHistory(ctx, name, 1) + if err != nil { + return err + } + if len(records) > 0 { + run = records[0].Run + } + } + if tail <= 0 { + tail = 200 + } + var cmd string + switch { + case run == "": + cmd = "journalctl -u " + q(unit+".service") + " -n " + strconv.Itoa(tail) + " --no-pager -o short-iso" + case scheduleRunID.MatchString(run): + cmd = "journalctl _SYSTEMD_INVOCATION_ID=" + run + " --no-pager -o short-iso" + default: + return fmt.Errorf("run id %q is not a systemd invocation id", run) + } + return e.T.RunStream(ctx, cmd, stdout, stderr) +} +``` + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine` +Expected: PASS. If `transport.Fake` lacks `RunStream`, check `internal/transport/fake.go`; it implements the interface, so this compiles. + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine/schedule_history.go internal/engine/schedule_history_test.go +git commit -m "feat(schedule): read run records, timer state and per-run logs from the host journal + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 4: `ob status` reads the record + +**Files:** +- Modify: `internal/engine/schedule_status.go` (struct, batched command, parser), `internal/engine/status.go:168-187` (human lines) +- Test: `internal/engine/schedule_test.go` + +**Interfaces:** +- Produces new `StatusSchedule` fields: `NextRun string json:"next_run,omitempty"`, `LastOutcome string json:"last_outcome,omitempty"`, `LastDurationSeconds int json:"last_duration_s,omitempty"`, `LastAttempts int json:"last_attempts,omitempty"`, `ConsecutiveFailures int json:"consecutive_failures,omitempty"`, `Attempt int json:"attempt,omitempty"`, `JournalPersistent bool json:"journal_persistent"`. +- A record with outcome `failure` or `timeout` produces the issue `last run failed: (exit N)`; `skipped` and `success` produce none. With no record, today's systemd-based issue stays. + +- [ ] **Step 1: Write the failing test** + +```go +func TestScheduleStatusPrefersTheRunRecordOverSystemdResult(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: `@@journal +persistent +@@nightly:service +LoadState=loaded +ActiveState=inactive +Result=success +ExecMainStatus=75 +@@nightly:timer +LoadState=loaded +ActiveState=active +NextElapseUSecRealtime=Sat 2026-09-06 02:00:00 UTC +@@nightly:run +@@nightly:history +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","release":"r1","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:02Z","duration_s":1,"attempts":0,"exit_status":75,"outcome":"skipped","inputs":{}} +{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"timer","release":"r1","started_at":"2026-09-04T02:00:01Z","finished_at":"2026-09-04T02:05:02Z","duration_s":301,"attempts":3,"exit_status":1,"outcome":"failure","inputs":{}} +{"run":"c3d4e5f60718293a4b5c6d7e8f901234","job":"nightly","trigger":"timer","release":"r1","started_at":"2026-09-03T02:00:01Z","finished_at":"2026-09-03T02:01:02Z","duration_s":61,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}} +`}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + got := statuses[0] + if got.LastOutcome != "skipped" || got.NextRun != "Sat 2026-09-06 02:00:00 UTC" || !got.JournalPersistent { + t.Fatalf("record fields not surfaced: %#v", got) + } + if got.ConsecutiveFailures != 0 || got.Diverged { + t.Fatalf("a skip is not a failure: %#v", got) + } + if got.LastAttempts != 0 || got.LastDurationSeconds != 1 { + t.Fatalf("last run detail not surfaced: %#v", got) + } +} + +func TestScheduleStatusCountsConsecutiveFailuresFromRecords(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: `@@journal +volatile +@@nightly:service +LoadState=loaded +ActiveState=failed +Result=exit-code +ExecMainStatus=1 +@@nightly:timer +LoadState=loaded +ActiveState=active +@@nightly:run +@@nightly:history +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:02Z","duration_s":1,"attempts":2,"exit_status":1,"outcome":"failure","inputs":{}} +{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"timer","started_at":"2026-09-04T02:00:01Z","finished_at":"2026-09-04T02:05:02Z","duration_s":301,"attempts":2,"exit_status":null,"outcome":"timeout","inputs":{}} +{"run":"c3d4e5f60718293a4b5c6d7e8f901234","job":"nightly","trigger":"timer","started_at":"2026-09-03T02:00:01Z","finished_at":"2026-09-03T02:01:02Z","duration_s":61,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}} +`}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + got := statuses[0] + if got.ConsecutiveFailures != 2 || !got.Diverged || got.JournalPersistent { + t.Fatalf("failures not counted: %#v", got) + } + if !strings.Contains(strings.Join(got.Issues, "; "), "last run failed: failure (exit 1)") { + t.Fatalf("issue does not name the outcome: %#v", got.Issues) + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/engine -run 'TestScheduleStatusPrefersTheRunRecord|TestScheduleStatusCountsConsecutive'` +Expected: FAIL (unknown fields). + +- [ ] **Step 3: Implement** + +`internal/engine/schedule_status.go`: add the fields to `StatusSchedule`; add `history []ScheduleRunRecord` and `next string` to `scheduleUnitObservation`; extend the batched command. Replace the loop body building `commands` with: + +```go + commands := []string{ + "printf '%s\\n' '@@journal'", + "if [ -d /var/log/journal ]; then echo persistent; else echo volatile; fi", + } + for _, job := range jobs { + unit := e.names().ScheduledJobUnit(job.Name) + commands = append(commands, + "printf '%s\\n' "+q("@@"+job.Name+":service"), + "systemctl show "+q(unit+".service")+" --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus", + "printf '%s\\n' "+q("@@"+job.Name+":timer"), + "systemctl show "+q(unit+".timer")+" --no-pager --property=LoadState --property=ActiveState --property=NextElapseUSecRealtime", + "printf '%s\\n' "+q("@@"+job.Name+":run"), + "cat "+q(e.names().ScheduledJobRunState(job.Name))+" 2>/dev/null || true", + "printf '%s\\n' "+q("@@"+job.Name+":history"), + scheduleHistoryCommand(unit, 20), + ) + } +``` + +Parser: keep `values` for `key=value` kinds; for kind `history` collect raw lines; for marker `@@journal` (no colon) read the next non-empty line into `journalPersistent`. Replace the parse loop and `flush` with: + +```go + observed := map[string]map[string]scheduleUnitObservation{} + journalPersistent := true + name, kind := "", "" + values := map[string]string{} + var raw []string + flush := func() { + if name == "" || kind == "" { + return + } + if observed[name] == nil { + observed[name] = map[string]scheduleUnitObservation{} + } + exit, _ := strconv.Atoi(values["ExecMainStatus"]) + observed[name][kind] = scheduleUnitObservation{ + loadState: values["LoadState"], activeState: values["ActiveState"], + result: values["Result"], exitStatus: exit, + release: values["release"], startedAt: values["started_at"], attempt: values["attempt"], + next: values["NextElapseUSecRealtime"], + history: parseScheduleRunRecords(strings.Join(raw, "\n")), + } + values = map[string]string{} + raw = nil + } + for _, line := range strings.Split(res.Stdout, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "@@") { + flush() + marker := strings.TrimPrefix(line, "@@") + if marker == "journal" { + name, kind = "", "" + continue + } + name, kind, _ = strings.Cut(marker, ":") + continue + } + if name == "" { + if line == "volatile" { + journalPersistent = false + } + continue + } + if kind == "history" { + raw = append(raw, line) + continue + } + if key, value, ok := strings.Cut(line, "="); ok { + values[key] = value + } + } + flush() +``` + +Add `attempt`, `next string` and `history []ScheduleRunRecord` to `scheduleUnitObservation`. In the status assembly loop, after building `status`: + +```go + status.JournalPersistent = journalPersistent + status.NextRun = timer.next + if status.Running { + status.Attempt, _ = strconv.Atoi(run.attempt) + } + records := observed[job.Name]["history"].history + if len(records) > 0 { + last := records[0] + status.LastOutcome = last.Outcome + status.LastDurationSeconds = last.DurationSeconds + status.LastAttempts = last.Attempts + for _, record := range records { + if record.Outcome == "skipped" { + continue + } + if record.Outcome != "failure" && record.Outcome != "timeout" { + break + } + status.ConsecutiveFailures++ + } + } +``` + +Replace the systemd-result issue block with: + +```go + switch { + case status.LastOutcome == "failure" || status.LastOutcome == "timeout": + exit := "?" + if records[0].ExitStatus != nil { + exit = strconv.Itoa(*records[0].ExitStatus) + } + status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %s)", status.LastOutcome, exit)) + case status.LastOutcome == "": + if service.result != "" && service.result != "success" { + status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %d)", service.result, service.exitStatus)) + } + } +``` + +`internal/engine/status.go` human lines, replace the final `fmt.Fprintf(... "active; policy: ...")` with: + +```go + detail := fmt.Sprintf("active; policy: %s; timeout: %s", schedule.DeployLock, schedule.Timeout) + if schedule.NextRun != "" { + detail += "; next: " + schedule.NextRun + } + last := result + if schedule.LastOutcome != "" { + last = fmt.Sprintf("%s (%ds, %d attempt(s))", schedule.LastOutcome, schedule.LastDurationSeconds, schedule.LastAttempts) + } + detail += "; last: " + last + if !schedule.JournalPersistent { + detail += "; journal: volatile, history since boot only" + } + fmt.Fprintf(e.Opts.Out, "schedule %-11s %s\n", schedule.Name, detail) +``` + +and in the running branch append `fmt.Sprintf("; attempt: %d", schedule.Attempt)` when `schedule.Attempt > 0`. + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine ./cmd/ob` +Expected: PASS. The existing `TestScheduleStatusSurfacesTheLastSystemdFailure` still passes because it has no history block. + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine/schedule_status.go internal/engine/status.go internal/engine/schedule_test.go +git commit -m "feat(status): read scheduled-run outcomes, attempts and next elapse from the journal + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 5: CLI: `ob schedule list`, `ob schedule history`, `ob schedule logs` + +**Files:** +- Create: `cmd/ob/schedule.go` +- Modify: `cmd/ob/ops.go:69-88` (move `schedule` group construction into `addScheduleCommands(root, g)` in the new file), `cmd/ob/output.go:76-118` (matrix), `cmd/ob/output_test.go:492-530` (wantMatrix), `site/src/content/docs/reference/policies.mdx:98-102` +- Test: `cmd/ob/output_test.go`, `cmd/ob/docs_test.go` (regenerated `cli.mdx`) + +**Interfaces:** +- Consumes: `engine.ScheduleList`, `engine.ScheduleHistory`, `engine.ScheduleLogs`, `loadAllLenient`, `connect`, `newUI`, `writeFiniteSuccess`, `writeStructuredReadFailure`, `writeStructuredCommandFailure`, `isStructuredOutput`. +- Produces matrix rows: `"ob schedule list": {finite_envelope, JSON}`, `"ob schedule history": {finite_envelope, JSON}`, `"ob schedule logs": {operator_passthrough, JSON, NDJSON}`. + +- [ ] **Step 1: Update the closed-matrix test first** + +In `cmd/ob/output_test.go` `wantMatrix` add: + +```go + "ob schedule list": {Class: "finite_envelope", JSON: true}, + "ob schedule history": {Class: "finite_envelope", JSON: true}, + "ob schedule logs": {Class: "operator_passthrough", JSON: true, NDJSON: true}, +``` + +Run: `go test ./cmd/ob -run TestLeafOutputMatrixIsClosedAndHasNoAliases` +Expected: FAIL (matrix changed). + +- [ ] **Step 2: Implement `cmd/ob/schedule.go`** + +Move the existing `scheduleCmd`/`scheduleApplyCmd` construction from `ops.go` into `addScheduleCommands(root *cobra.Command, g *globalFlags)` and call it from where `root.AddCommand(scheduleCmd)` was. Then add: + +```go + var listCmd = &cobra.Command{ + Use: "list", + Short: "declared scheduled jobs with timer state and next elapse", + Long: "List every job that declares a schedule beside what the host's timer says: whether it is active, when it fires next, and when it last fired. Reads only.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, p, err := loadAllLenient(cmd.Context(), g) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + defer cleanup() + jobs, err := e.ScheduleList(cmd.Context()) + if err != nil { + return writeStructuredCommandFailure(cmd, g, "schedule_list_failed", "scheduled jobs could not be listed", err) + } + if isStructuredOutput(g) { + return writeFiniteSuccess(cmd, g, map[string]any{"jobs": jobs}) + } + if len(jobs) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "no scheduled jobs declared") + return nil + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "JOB\tCRON\tTZ\tTIMER\tNEXT\tLAST TRIGGER\tPOLICY\tTIMEOUT") + for _, j := range jobs { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", j.Name, j.Cron, j.Timezone, orDash(j.TimerState), orDash(j.NextRun), orDash(j.LastTrigger), j.DeployLock, j.Timeout) + } + return w.Flush() + }, + } +``` + +History: + +```go + var historyCount int + historyCmd := &cobra.Command{ + Use: "history ", + Short: "run records of one scheduled job, newest first", + Long: "Read the run records the host wrote for one scheduled job. Each record is one activation: run id, trigger, release, start and end, attempts, exit status, outcome and, for a manual run, its inputs.\n\nRecords live in the host journal under the job's unit with syslog identifier ob-run; retention is the journal's. Reads only.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, p, err := loadAllLenient(cmd.Context(), g) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + defer cleanup() + records, err := e.ScheduleHistory(cmd.Context(), args[0], historyCount) + if err != nil { + return writeStructuredCommandFailure(cmd, g, "schedule_history_failed", "run history could not be read", err) + } + if isStructuredOutput(g) { + return writeFiniteSuccess(cmd, g, map[string]any{"job": args[0], "runs": records}) + } + if len(records) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "no recorded runs for %s\n", args[0]) + return nil + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "STARTED\tOUTCOME\tDURATION\tATTEMPTS\tEXIT\tTRIGGER\tRELEASE\tRUN") + for _, r := range records { + exit := "-" + if r.ExitStatus != nil { + exit = strconv.Itoa(*r.ExitStatus) + } + fmt.Fprintf(w, "%s\t%s\t%ds\t%d\t%s\t%s\t%s\t%s\n", r.StartedAt, r.Outcome, r.DurationSeconds, r.Attempts, exit, r.Trigger, orDash(r.Release), r.Run) + } + return w.Flush() + }, + } + historyCmd.Flags().IntVarP(&historyCount, "count", "n", 20, "number of newest runs to show") +``` + +Logs: mirror `ob logs` in `cmd/ob/ops.go:229-300` exactly (JSON buffers `stdout`/`stderr` into `{"job","run","stdout","stderr","passthrough_unredacted":true}`; NDJSON uses the same streaming writer `ob logs` uses; human streams to the terminal). Flags: `--run ` and `-n/--tail` (default 200). Failure code `schedule_logs_failed`, message `run logs could not be read`. + +`orDash`: + +```go +func orDash(s string) string { + if strings.TrimSpace(s) == "" { + return "-" + } + return s +} +``` + +Add the three matrix rows to `cliOutputMatrix` in `output.go`. In `policies.mdx` add `` `ob schedule history` `` and `` `ob schedule list` `` to the Finite envelope row (alphabetical position after `ob preview`) and `` `ob schedule logs` `` to the `Operator passthrough | finite only | yes` row beside `ob logs`. + +- [ ] **Step 3: Regenerate docs and run tests** + +Run: `just build && just docs-generate && go test ./cmd/ob ./internal/... && just docs-generate-check` +Expected: PASS; `cli.mdx` gains the three commands. + +- [ ] **Step 4: Commit** + +```bash +git add cmd/ob/schedule.go cmd/ob/ops.go cmd/ob/output.go cmd/ob/output_test.go site/src/content/docs/reference/policies.mdx site/src/content/docs/reference/cli.mdx +git commit -m "feat(cli): ob schedule list, history and logs read the host's run records + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +## Slice 2: Retry and notify + +### Task 6: Schema, validation and defaults for `retry` and `notify` + +**Files:** +- Modify: `internal/app/types.go:268-274` (`JobSchedule`), `internal/app/constraints.go:173` (add `eScheduleNotify`), `internal/app/validate.go:573-584` (`validateJobSchedule`), `internal/app/schedule.go:26-64` (`ScheduledJob`, `ScheduledJobs`) +- Test: `internal/app/schedule_test.go` + +**Interfaces:** +- Produces: + - `type JobRetry struct { Attempts int; Backoff string; MaxBackoff string }` JSON `attempts`, `backoff`, `max_backoff`, with `description` and `default` tags. + - `JobSchedule.Retry *JobRetry json:"retry,omitempty"`, `JobSchedule.Notify []string json:"notify,omitempty"`. + - `ScheduledJob` gains `RetryAttempts int`, `RetryBackoff time.Duration`, `RetryMaxBackoff time.Duration`, `Notify []string`, resolved with defaults in `ScheduledJobs()`. + - `func scheduleRetryWorstCase(attempts int, backoff, max time.Duration) time.Duration` = sum over the attempts-1 sleeps of `min(backoff·2^i, max)`. + +- [ ] **Step 1: Write the failing tests** + +Append to `internal/app/schedule_test.go`: + +```go +func TestScheduledJobRetryAndNotifyResolveWithDefaults(t *testing.T) { + spec, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + plain: + role: job + image: x:1 + data_effect: none + schedule: {cron: "0 3 * * *"} + retrying: + role: job + image: x:1 + data_effect: none + schedule: + cron: "0 * * * *" + timeout: 45m + retry: {attempts: 3, backoff: 30s, max_backoff: 10m} + notify: [failure, timeout, skipped] +`), "ob.yml") + if err != nil { + t.Fatal(err) + } + jobs, err := spec.ScheduledJobs() + if err != nil { + t.Fatal(err) + } + byName := map[string]ScheduledJob{} + for _, job := range jobs { + byName[job.Name] = job + } + plain := byName["plain"] + if plain.RetryAttempts != 1 || plain.RetryBackoff != 30*time.Second || plain.RetryMaxBackoff != 10*time.Minute || + strings.Join(plain.Notify, ",") != "failure,timeout" { + t.Fatalf("defaults did not resolve: %#v", plain) + } + retrying := byName["retrying"] + if retrying.RetryAttempts != 3 || strings.Join(retrying.Notify, ",") != "failure,timeout,skipped" { + t.Fatalf("declared retry did not resolve: %#v", retrying) + } +} + +func TestScheduledJobRetryIsBoundedByTheTimeout(t *testing.T) { + for name, tc := range map[string]struct { + schedule string + code string + }{ + "too many attempts": {`{cron: "0 * * * *", retry: {attempts: 11}}`, "project_invalid"}, + "zero attempts": {`{cron: "0 * * * *", retry: {attempts: 0}}`, "project_invalid"}, + "backoff over max": {`{cron: "0 * * * *", retry: {attempts: 2, backoff: 20m, max_backoff: 10m}}`, "project_invalid"}, + "backoff exceeds timeout": {`{cron: "0 * * * *", timeout: 5m, retry: {attempts: 3, backoff: 2m, max_backoff: 30m}}`, "project_invalid"}, + "unknown notify": {`{cron: "0 * * * *", notify: [warning]}`, "project_invalid"}, + } { + t.Run(name, func(t *testing.T) { + _, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + j: {role: job, image: x:1, data_effect: none, schedule: `+tc.schedule+`} +`), "ob.yml") + var e *Error + if !errors.As(err, &e) || e.Code != tc.code { + t.Fatalf("err = %v, want code %s", err, tc.code) + } + }) + } + if got := scheduleRetryWorstCase(3, 2*time.Minute, 30*time.Minute); got != 6*time.Minute { + t.Fatalf("worst case = %s, want 6m", got) + } + if got := scheduleRetryWorstCase(4, 30*time.Second, time.Minute); got != 150*time.Second { + t.Fatalf("capped worst case = %s, want 2m30s", got) + } +} +``` + +Add `"errors"`, `"strings"`, `"time"` to that test file's imports if absent. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/app -run 'TestScheduledJobRetry'` +Expected: FAIL to compile. + +- [ ] **Step 3: Implement** + +`types.go`: + +```go +type JobSchedule struct { + Cron string `json:"cron" ...unchanged...` + Timezone string ... + Timeout string ... + CatchUp bool ... + DeployLock string ... + Retry *JobRetry `json:"retry,omitempty" description:"Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run."` + Notify []string `json:"notify,omitempty" description:"Run outcomes that send the configured notifications: success, failure, timeout, skipped." default:"failure, timeout"` +} + +type JobRetry struct { + Attempts int `json:"attempts,omitempty" description:"Total attempts including the first, 1 to 10." default:"1" example:"3"` + Backoff string `json:"backoff,omitempty" description:"Sleep before the second attempt; it doubles after each failure." default:"30s" example:"1m"` + MaxBackoff string `json:"max_backoff,omitempty" description:"Upper bound for the doubling sleep." default:"10m" example:"30m"` +} +``` + +`constraints.go` beside `eNotifyEvent`: `eScheduleNotify = []string{"success", "failure", "timeout", "skipped"}`. + +`schedule.go`: add constants and resolution: + +```go +const ( + defaultRetryAttempts = 1 + defaultRetryBackoff = 30 * time.Second + defaultRetryMaxBackoff = 10 * time.Minute +) + +var defaultScheduleNotify = []string{"failure", "timeout"} + +// scheduleRetryWorstCase is the longest a run can spend asleep between +// attempts. Validation keeps it under the timeout, so the last attempt can +// always start. +func scheduleRetryWorstCase(attempts int, backoff, max time.Duration) time.Duration { + var total time.Duration + sleep := backoff + for i := 1; i < attempts; i++ { + if sleep > max { + sleep = max + } + total += sleep + sleep *= 2 + } + return total +} + +func (s *JobSchedule) retryPolicy() (int, time.Duration, time.Duration) { + attempts, backoff, max := defaultRetryAttempts, defaultRetryBackoff, defaultRetryMaxBackoff + if s.Retry != nil { + if s.Retry.Attempts > 0 { + attempts = s.Retry.Attempts + } + if d, ok := ParseDuration(s.Retry.Backoff); ok && s.Retry.Backoff != "" { + backoff = d + } + if d, ok := ParseDuration(s.Retry.MaxBackoff); ok && s.Retry.MaxBackoff != "" { + max = d + } + } + return attempts, backoff, max +} +``` + +Extend `ScheduledJob` with `RetryAttempts int`, `RetryBackoff, RetryMaxBackoff time.Duration`, `Notify []string`; in `ScheduledJobs()` fill them: + +```go + attempts, backoff, max := w.Schedule.retryPolicy() + notify := w.Schedule.Notify + if len(notify) == 0 { + notify = append([]string(nil), defaultScheduleNotify...) + } + out = append(out, ScheduledJob{ + Name: name, Cron: w.Schedule.Cron, Timezone: tz, Calendar: cal, + Timeout: w.Schedule.Timeout, CatchUp: w.Schedule.CatchUp, DeployLock: deployLock, + RetryAttempts: attempts, RetryBackoff: backoff, RetryMaxBackoff: max, Notify: notify, + }) +``` + +`validate.go` `validateJobSchedule`, after the timeout check: + +```go + if s.Retry != nil { + if s.Retry.Attempts < 0 || s.Retry.Attempts > 10 { + return errf("project_invalid", path+".retry.attempts", "", "attempts must be between 1 and 10, got %d", s.Retry.Attempts) + } + if raw, err := yamlZeroAttempts(s); err == nil && raw { + return errf("project_invalid", path+".retry.attempts", "", "attempts must be between 1 and 10, got 0") + } + if err := gDur.checkOptional(path+".retry.backoff", s.Retry.Backoff); err != nil { + return err + } + if err := gDur.checkOptional(path+".retry.max_backoff", s.Retry.MaxBackoff); err != nil { + return err + } + attempts, backoff, max := s.retryPolicy() + if backoff > max { + return errf("project_invalid", path+".retry.backoff", "", "backoff %s exceeds max_backoff %s", s.Retry.Backoff, s.Retry.MaxBackoff) + } + timeout := time.Hour + if s.Timeout != "" { + if d, ok := ParseDuration(s.Timeout); ok { + timeout = d + } + } + if worst := scheduleRetryWorstCase(attempts, backoff, max); worst >= timeout { + return errf("project_invalid", path+".retry", "", + "worst-case backoff %s is not smaller than timeout %s; later attempts could never start", worst, timeout) + } + } + for i, outcome := range s.Notify { + if err := checkEnum(indexed(path+".notify", i), outcome, eScheduleNotify); err != nil { + return err + } + } +``` + +`attempts: 0` cannot be told from "absent" after decoding into `int`. Make `Attempts` a `*int`? Simpler and honest: change `JobRetry.Attempts` to `int` and treat `0` as "not set" only when the whole `retry` block is absent; when `retry:` is present with `attempts: 0`, the raw YAML check `applyDefaults` already receives `raw map[string]any`. Rather than a raw lookup, declare `Attempts *int` with `default:"1"` and validate `*Attempts` in 1..10 when non-nil; `retryPolicy` uses `*Attempts` when non-nil. Delete the `yamlZeroAttempts` line above and implement with the pointer. `schemaTagValue` handles `*int` via `deref` for the default tag; verify with `TestPublishedSchemaDocumentsEveryPublicField`. + +- [ ] **Step 4: Run tests and regenerate schema** + +Run: `go test ./internal/app && go run ./cmd/ob schema --out docs/onebox.run-v1.schema.json && go test ./internal/app ./cmd/ob-docgen` +Expected: PASS. `TestContractDidNotMove` still passes: no existing case declares `retry` or `notify`, and neither field renders into Compose. + +- [ ] **Step 5: Commit** + +```bash +git add internal/app docs/onebox.run-v1.schema.json +git commit -m "feat(schema): schedule.retry and schedule.notify with validation against the timeout + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 7: Runner retry loop and per-outcome notifications + +**Files:** +- Modify: `internal/engine/schedule.go` (`scheduleRunnerScript`, `pinnedScheduleRunnerScript`, `scheduleFailureNotifier`) +- Test: `internal/engine/schedule_test.go` + +**Interfaces:** +- Produces: `func scheduleAttemptLoop(job app.ScheduledJob, compose string) []string`; notifier renders a success body and a failure body per notification and selects by outcome; `scheduleFailureNotifier(job string)` becomes `scheduleNotifier(job app.ScheduledJob)`. + +- [ ] **Step 1: Write the failing tests** + +```go +func TestScheduledJobRunnerRetriesWithCappedDoublingBackoff(t *testing.T) { + job := app.ScheduledJob{Name: "nightly", Timeout: "45m", DeployLock: "exclusive", + RetryAttempts: 3, RetryBackoff: 30 * time.Second, RetryMaxBackoff: 10 * time.Minute} + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil) + for _, want := range []string{ + "max_attempts=3", "backoff=30", "max_backoff=600", + "attempt=1", "while :; do", "write_state \"$attempt\"", + "status=0", "|| status=$?", "[ \"$status\" -eq 0 ] && exit 0", + "if [ \"$attempt\" -ge \"$max_attempts\" ]; then exit \"$status\"; fi", + "sleep \"$backoff\"", "backoff=$((backoff * 2))", "attempt=$((attempt + 1))", + } { + if !strings.Contains(runner, want) { + t.Errorf("runner is missing %q:\n%s", want, runner) + } + } + single := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil) + if strings.Contains(single, "while :; do") { + t.Errorf("a single-attempt job must not carry a retry loop:\n%s", single) + } + for _, script := range []string{runner, single} { + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(script) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("runner is not valid POSIX shell: %v: %s\n%s", err, output, script) + } + } +} + +func TestScheduledJobNotifierSelectsBodiesByOutcome(t *testing.T) { + cfg := testConfig() + cfg.Notifications = map[string]app.Notification{ + "ops": {Webhook: "https://hooks.example.com/ops", On: []string{"success", "failure"}, Format: "json"}, + } + f := &transport.Fake{TargetName: "root@example.internal"} + e := New(cfg, testProject(t), f, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + script, err := e.scheduleNotifier(app.ScheduledJob{Name: "nightly", Notify: []string{"success", "failure", "skipped"}}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `case " success failure skipped " in *" $outcome "*) ;; *) exit 0 ;; esac`, + `"status":"ok"`, + `"status":"fail"`, + `if [ "$outcome" = success ]; then`, + `"$detail"`, + `detail="run ${INVOCATION_ID:-?}: $outcome after $attempt attempt(s) in ${duration}s (exit $status)"`, + } { + if !strings.Contains(script, want) { + t.Errorf("notifier is missing %q:\n%s", want, script) + } + } +} +``` + +Update `TestScheduledJobFailureNotifierUsesConfiguredWebhooks` to call `e.scheduleNotifier(app.ScheduledJob{Name: "nightly", Notify: []string{"failure", "timeout"}})` and keep its expectations; the `success-only` webhook is still absent because its `On` excludes failure and the job's `Notify` excludes success. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/engine -run 'TestScheduledJobRunnerRetries|TestScheduledJobNotifierSelects'` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +`scheduleAttemptLoop`: + +```go +// scheduleAttemptLoop runs the container until it exits 0 or the attempts are +// spent. Backoff doubles and is capped; every sleep happens under the locks the +// run already holds, which is why validation keeps the sum under the timeout. +func scheduleAttemptLoop(job app.ScheduledJob, compose string) []string { + if job.RetryAttempts <= 1 { + return []string{"write_state 1", compose} + } + return []string{ + fmt.Sprintf("max_attempts=%d", job.RetryAttempts), + fmt.Sprintf("backoff=%d", int(math.Ceil(job.RetryBackoff.Seconds()))), + fmt.Sprintf("max_backoff=%d", int(math.Ceil(job.RetryMaxBackoff.Seconds()))), + "attempt=1", + "while :; do", + " write_state \"$attempt\"", + " status=0", + " " + compose + " || status=$?", + " [ \"$status\" -eq 0 ] && exit 0", + " if [ \"$attempt\" -ge \"$max_attempts\" ]; then exit \"$status\"; fi", + " echo \"onebox: attempt $attempt of $max_attempts exited $status; retrying in ${backoff}s\" >&2", + " sleep \"$backoff\"", + " backoff=$((backoff * 2))", + " [ \"$backoff\" -gt \"$max_backoff\" ] && backoff=$max_backoff", + " attempt=$((attempt + 1))", + "done", + } +} +``` + +Both runners: replace `lines = append(lines, "write_state 1", compose, "")` with `lines = append(lines, scheduleAttemptLoop(job, compose)...); lines = append(lines, "")` (pinned keeps `"/usr/bin/flock --unlock 8"` before the loop). `pinnedScheduleRunnerScript` takes `job app.ScheduledJob` instead of `job string`; update its two callers. + +Notifier: rename to `scheduleNotifier(job app.ScheduledJob)`; after the record lines: + +```go + lines = append(lines, + `detail="run ${INVOCATION_ID:-?}: $outcome after $attempt attempt(s) in ${duration}s (exit $status)"`, + `case " `+strings.Join(job.Notify, " ")+` " in *" $outcome "*) ;; *) exit 0 ;; esac`, + ) +``` + +For each notification prepare two payloads: `Status: "ok", Error: ""` and `Status: "fail", Error: scheduleNotificationDetail` where `const scheduleNotificationDetail = "__ONEBOX_SCHEDULE_DETAIL__"`; substitute both `scheduleNotificationTimestamp` with `"$ts"` and `scheduleNotificationDetail` with `"$detail"` (same `strings.Cut` technique, applied twice through a small helper `shellBody(body string) string`). Emit: + +``` +ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ') +if [ "$outcome" = success ]; then + +else + +fi +wait || true +exit 0 +``` + +Both branches may be empty; render `:` in an empty branch so the shell stays valid. The verb stays `"scheduled job " + job.Name`; the `Text` line for text-format webhooks carries the detail through the same placeholder. + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine ./cmd/...` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine +git commit -m "feat(schedule): bounded retry in the runner and per-outcome notifications + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +## Slice 3: Inputs and manual runs + +### Task 8: Schema and validation for `inputs`; defaults rendered into Compose + +**Files:** +- Modify: `internal/app/types.go:136-174` (`Workload.Inputs`), `internal/app/validate.go:436-458`, `internal/app/generate.go:347` (environment), `internal/app/schedule.go` (`ScheduledJob.Inputs`), `internal/app/constraints.go`, `internal/app/names.go:242` +- Test: `internal/app/schedule_test.go`, `internal/app/generate_test.go` (or the nearest existing render test file) + +**Interfaces:** +- Produces: + - `type JobInput struct { Enum []string; Pattern string; Default string; Description string }` JSON `enum, pattern, default, description`. + - `Workload.Inputs map[string]JobInput json:"inputs,omitempty"`. + - `ScheduledJob.Inputs map[string]JobInput`. + - `func ValidateJobInputValues(w Workload, values map[string]string) error` (used by the engine for overrides): unknown name, pattern/enum mismatch, charset, length. + - `var gInputName = grammar{"input name", ^[A-Z][A-Z0-9_]*$}`, `const ReservedInputPrefix = "ONEBOX_"`, `func InputValueAllowed(v string) bool` (≤256 bytes, no `"`, `\`, control chars). + - `func (n Names) ScheduledJobRunInputs(job string) string` = `/schedule/.inputs`. + +- [ ] **Step 1: Write the failing tests** + +```go +func TestJobInputsValidateNamesConstraintsAndDefaults(t *testing.T) { + base := `api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + sync: + role: job + image: x:1 + data_effect: %s + env: {MODE: fast} + %s + inputs: + %s +` + load := func(effect, schedule, inputs string) error { + _, err := LoadBytes([]byte(fmt.Sprintf(base, effect, schedule, inputs)), "ob.yml") + return err + } + good := "SOURCE: {enum: [catalog, prices], default: catalog, description: Which upstream.}" + if err := load("none", `schedule: {cron: "0 * * * *"}`, good); err != nil { + t.Fatalf("valid inputs refused: %v", err) + } + for name, tc := range map[string]struct{ effect, schedule, inputs string }{ + "no schedule": {"none", "", good}, + "destructive job": {"destructive", `schedule: {cron: "0 * * * *"}`, good}, + "lowercase name": {"none", `schedule: {cron: "0 * * * *"}`, "source: {enum: [a], default: a}"}, + "reserved prefix": {"none", `schedule: {cron: "0 * * * *"}`, "ONEBOX_X: {enum: [a], default: a}"}, + "collides with env": {"none", `schedule: {cron: "0 * * * *"}`, "MODE: {enum: [a], default: a}"}, + "enum and pattern": {"none", `schedule: {cron: "0 * * * *"}`, "S: {enum: [a], pattern: '^a$', default: a}"}, + "neither": {"none", `schedule: {cron: "0 * * * *"}`, "S: {default: a}"}, + "default off enum": {"none", `schedule: {cron: "0 * * * *"}`, "S: {enum: [a], default: b}"}, + "default off pattern": {"none", `schedule: {cron: "0 * * * *"}`, "S: {pattern: '^[0-9]+$', default: x}"}, + "quote in default": {"none", `schedule: {cron: "0 * * * *"}`, `S: {pattern: '.*', default: 'a"b'}`}, + "bad regex": {"none", `schedule: {cron: "0 * * * *"}`, "S: {pattern: '(', default: a}"}, + } { + t.Run(name, func(t *testing.T) { + var e *Error + if err := load(tc.effect, tc.schedule, tc.inputs); !errors.As(err, &e) || e.Code != "project_invalid" { + t.Fatalf("err = %v, want project_invalid", err) + } + }) + } +} + +func TestValidateJobInputValuesChecksOverrides(t *testing.T) { + w := Workload{Role: RoleJob, Inputs: map[string]JobInput{ + "SOURCE": {Enum: []string{"catalog", "prices"}, Default: "catalog"}, + "SINCE": {Pattern: `^([0-9]{4}-[0-9]{2}-[0-9]{2})?$`, Default: ""}, + }} + if err := ValidateJobInputValues(w, map[string]string{"SOURCE": "prices", "SINCE": "2026-09-01"}); err != nil { + t.Fatal(err) + } + for name, values := range map[string]map[string]string{ + "unknown": {"OTHER": "x"}, + "off enum": {"SOURCE": "reviews"}, + "off pattern": {"SINCE": "yesterday"}, + "backslash": {"SINCE": `2026\-09-01`}, + "newline": {"SOURCE": "prices\n"}, + } { + if err := ValidateJobInputValues(w, values); err == nil { + t.Errorf("%s was accepted", name) + } + } + if InputValueAllowed(strings.Repeat("a", 257)) { + t.Error("an oversized value was allowed") + } +} + +func TestScheduledJobInputDefaultsRenderIntoTheComposeEnvironment(t *testing.T) { + spec, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + sync: + role: job + image: x:1 + data_effect: none + env: {MODE: fast} + schedule: {cron: "0 * * * *"} + inputs: + SOURCE: {enum: [catalog, prices], default: catalog} +`), "ob.yml") + if err != nil { + t.Fatal(err) + } + rendered := renderForTest(t, spec) // use the same helper the existing generate tests use to render Compose YAML + for _, want := range []string{"SOURCE: catalog", "MODE: fast"} { + if !strings.Contains(rendered, want) { + t.Fatalf("rendered runtime is missing %q:\n%s", want, rendered) + } + } + jobs, _ := spec.ScheduledJobs() + if jobs[0].Inputs["SOURCE"].Default != "catalog" { + t.Fatalf("scheduled job did not carry its inputs: %#v", jobs[0]) + } +} +``` + +Before writing the render test, locate the render helper with `rtk rg -n 'func render.*\(t \*testing.T' internal/app/*_test.go` and use its real name in place of `renderForTest`. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/app -run 'TestJobInputs|TestValidateJobInputValues|TestScheduledJobInputDefaults'` +Expected: FAIL to compile. + +- [ ] **Step 3: Implement** + +`types.go`, in `Workload` under `// Job only.`: + +```go + Inputs map[string]JobInput `json:"inputs,omitempty" description:"Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them."` +``` + +and + +```go +type JobInput struct { + Enum []string `json:"enum,omitempty" description:"Accepted values." example:"catalog"` + Pattern string `json:"pattern,omitempty" description:"Regular expression the whole value must match." example:"^[0-9]{4}-[0-9]{2}-[0-9]{2}$"` + Default string `json:"default" description:"Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint."` + Description string `json:"description,omitempty" description:"What the input controls."` +} +``` + +`constraints.go`: + +```go + gInputName = grammar{"input name", regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`), + "upper-case letters, digits and underscores, starting with a letter"} +``` + +(match the existing `grammar` literal shape; the third field is the hint string if the struct has one). + +`schedule.go` (app): + +```go +// ReservedInputPrefix is Onebox's environment namespace; declared inputs may +// not enter it, and the manual-run metadata line lives there. +const ReservedInputPrefix = "ONEBOX_" + +const maxInputValueBytes = 256 + +// InputValueAllowed is the charset that lets the runner pass a value through +// to the container and into the run record without escaping anything. +func InputValueAllowed(v string) bool { + if len(v) > maxInputValueBytes { + return false + } + for _, r := range v { + if r == '"' || r == '\\' || r < 0x20 || r == 0x7f { + return false + } + } + return true +} + +func (in JobInput) accepts(v string) bool { + if !InputValueAllowed(v) { + return false + } + if len(in.Enum) > 0 { + for _, allowed := range in.Enum { + if v == allowed { + return true + } + } + return false + } + re, err := regexp.Compile(in.Pattern) + if err != nil { + return false + } + return re.MatchString(v) && re.FindStringIndex(v) != nil && wholeMatch(re, v) +} + +func wholeMatch(re *regexp.Regexp, v string) bool { + loc := re.FindStringIndex(v) + return loc != nil && loc[0] == 0 && loc[1] == len(v) +} + +// ValidateJobInputValues checks operator-supplied overrides against the +// declaration. Defaults were checked at load; this is the other half. +func ValidateJobInputValues(w Workload, values map[string]string) error { + for _, name := range sortedKeys(values) { + in, ok := w.Inputs[name] + if !ok { + return fmt.Errorf("input %s is not declared", name) + } + if !in.accepts(values[name]) { + return fmt.Errorf("input %s: %q is not an accepted value", name, values[name]) + } + } + return nil +} +``` + +(`sortedKeys` is generic over map value type in this package? Check its signature at `internal/app/schedule.go` usage; if it is `map[string]Workload`-specific, add a local `sortedStringKeys(map[string]string)`.) In `ScheduledJobs()` add `Inputs: w.Inputs` to the literal and `Inputs map[string]JobInput` to `ScheduledJob`. + +`validate.go` inside `if w.IsJob() {` after the pinned block: + +```go + if len(w.Inputs) > 0 { + if w.Schedule == nil { + return errf("project_invalid", path+".inputs", "", "inputs belong to a scheduled job; declare schedule or remove inputs") + } + if w.DataEffect != DataEffectNone { + return errf("project_invalid", path+".inputs", "", + "inputs require data_effect %q; a %q job keeps the sealed plan of ob job run for operator-initiated runs", DataEffectNone, w.DataEffect) + } + for _, name := range sortedKeys(w.Inputs) { + ip := path + ".inputs." + name + if err := gInputName.check(ip, name); err != nil { + return err + } + if strings.HasPrefix(name, ReservedInputPrefix) { + return errf("project_invalid", ip, "", "%s is in Onebox's environment namespace", ReservedInputPrefix) + } + if _, clash := w.Env[name]; clash { + return errf("project_invalid", ip, "", "input %s collides with an env key of the same name", name) + } + in := w.Inputs[name] + switch { + case len(in.Enum) > 0 && in.Pattern != "": + return errf("project_invalid", ip, "", "declare exactly one of enum or pattern") + case len(in.Enum) == 0 && in.Pattern == "": + return errf("project_invalid", ip, "", "declare exactly one of enum or pattern") + } + if in.Pattern != "" { + if _, err := regexp.Compile(in.Pattern); err != nil { + return errf("project_invalid", ip+".pattern", "", "%v", err) + } + } + for i, v := range in.Enum { + if !InputValueAllowed(v) { + return errf("project_invalid", indexed(ip+".enum", i), "", "enum values may not contain quotes, backslashes or control characters and are at most %d bytes", maxInputValueBytes) + } + } + if !in.accepts(in.Default) { + return errf("project_invalid", ip+".default", "", "default %q does not satisfy the input's own constraint", in.Default) + } + } + } +``` + +Also in the `else if` branch for non-jobs add `|| len(w.Inputs) > 0` to the condition and mention `inputs` in the message. + +`generate.go:347`: + +```go + env := stringMap(w.Env) + if len(w.Inputs) > 0 { + if env == nil { + env = map[string]any{} + } + for name, in := range w.Inputs { + env[name] = in.Default + } + } + if len(env) > 0 { + svc["environment"] = env + } +``` + +`names.go`: + +```go +// ScheduledJobRunInputs is the one-shot file ob schedule run leaves for the +// next manual activation. The runner consumes and deletes it. +func (n Names) ScheduledJobRunInputs(job string) string { + return path.Join(n.AppDir(), "schedule", job+".inputs") +} +``` + +- [ ] **Step 4: Run tests, regenerate schema** + +Run: `go test ./internal/app && go run ./cmd/ob schema --out docs/onebox.run-v1.schema.json && go test ./internal/app ./cmd/ob-docgen` +Expected: PASS. If `TestContractDidNotMove` fails, no existing case uses `inputs`, so investigate before ever running `-update`. + +- [ ] **Step 5: Commit** + +```bash +git add internal/app docs/onebox.run-v1.schema.json +git commit -m "feat(schema): declared inputs for scheduled jobs, rendered as environment defaults + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 9: Runner consumes manual inputs; host systemd floor + +**Files:** +- Modify: `internal/engine/schedule.go` (both runners, `SyncSchedules`) +- Test: `internal/engine/schedule_test.go` + +**Interfaces:** +- Produces: `func scheduleInputsLines(inputsPath string) []string` inserted after the preamble in both runners; the compose command gains `"$@"` after `run --rm --no-deps`; `SyncSchedules` checks `systemctl --version` ≥ 252 when any job has inputs; error text: `job %s declares inputs, which need systemd 252 or newer on the host for $TRIGGER_UNIT; the host runs %s`. + +- [ ] **Step 1: Write the failing tests** + +```go +func TestScheduledJobRunnerConsumesManualInputsWithoutShellInterpolation(t *testing.T) { + job := app.ScheduledJob{Name: "sync", Timeout: "45m", DeployLock: "pinned", RetryAttempts: 1, + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog"}, Default: "catalog"}}} + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil) + for _, want := range []string{ + "inputs_file='/var/lib/ob/sample/schedule/sync.inputs'", + `if [ -z "${TRIGGER_UNIT:-}" ] && [ -f "$inputs_file" ]; then`, + `while IFS= read -r line || [ -n "$line" ]; do`, + `ONEBOX_OPERATION=*) operation=${line#ONEBOX_OPERATION=} ;;`, + `[A-Z]*=*) set -- "$@" -e "$line"`, + `rm -f "$inputs_file"`, + `run --rm --no-deps "$@" --name 'sample-sync-1' 'sync'`, + } { + if !strings.Contains(runner, want) { + t.Errorf("runner is missing %q:\n%s", want, runner) + } + } + if strings.Contains(runner, ". \"$inputs_file\"") || strings.Contains(runner, "eval") { + t.Fatalf("runner evaluates the inputs file as shell:\n%s", runner) + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(runner) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("runner is not valid POSIX shell: %v: %s\n%s", err, output, runner) + } + // The consume block precedes the locks so a skipped manual run cannot + // leave its inputs for the next timer firing. + if strings.Index(runner, "inputs_file=") > strings.Index(runner, "flock --exclusive --nonblock --conflict-exit-code 75 9") { + t.Fatalf("inputs are consumed after the lock:\n%s", runner) + } +} + +func TestSyncSchedulesRequiresSystemd252ForInputs(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"a"}, Default: "a"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "list-unit-files"): + return transport.Result{}, true + case strings.Contains(cmd, "systemd-analyze calendar"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl --version"): + return transport.Result{Stdout: "systemd 249 (249.11-0ubuntu3)\n"}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + err := e.SyncSchedules(context.Background()) + if err == nil || !strings.Contains(err.Error(), "systemd 252") { + t.Fatalf("old systemd was accepted for a job with inputs: %v", err) + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/engine -run 'TestScheduledJobRunnerConsumesManualInputs|TestSyncSchedulesRequiresSystemd252'` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +```go +// scheduleInputsLines consumes the one-shot inputs file on a manual +// activation. Values reach the container as -e arguments, never as shell +// text, and the file is gone before any lock is taken so a skipped manual run +// cannot hand its inputs to the next timer firing. +func scheduleInputsLines(inputsPath string) []string { + return []string{ + "inputs_file=" + q(inputsPath), + "if [ -z \"${TRIGGER_UNIT:-}\" ] && [ -f \"$inputs_file\" ]; then", + " while IFS= read -r line || [ -n \"$line\" ]; do", + " case \"$line\" in", + " ONEBOX_OPERATION=*) operation=${line#ONEBOX_OPERATION=} ;;", + " [A-Z]*=*) set -- \"$@\" -e \"$line\"; key=${line%%=*}; value=${line#*=}; inputs_json=\"${inputs_json:+$inputs_json,}\\\"$key\\\":\\\"$value\\\"\" ;;", + " esac", + " done <\"$inputs_file\"", + " rm -f \"$inputs_file\"", + "fi", + } +} +``` + +Order inside both runners: shebang, `set -eu`, `install -d`, then `operation=''`, `inputs_json=''`, then `scheduleInputsLines(...)`, then the flocks and the rest; move the two variable initialisations out of `scheduleRunPreamble` (keep the preamble's `state`, `tmp`, `started_*`, `trigger`, `write_state`). The compose command string in both runners becomes: + +```go + compose := "/usr/bin/docker compose -p " + q(application) + " --project-directory " + projectDir + + " -f " + projectDir + "/" + q("compose.yaml") + scheduleRuntimeEnvArgs(projectDir, runtimeEnvFiles) + + " run --rm --no-deps \"$@\" --name " + q(container) + " " + q(job.Name) +``` + +In `SyncSchedules`, before the per-job loop: + +```go + if needsTriggerUnit(jobs) { + res, err := e.T.Run(ctx, "systemctl --version 2>/dev/null | head -1") + if err != nil { + return err + } + if version, ok := systemdVersion(res.Stdout); !ok || version < 252 { + return fmt.Errorf("a job declares inputs, which need systemd 252 or newer on the host for $TRIGGER_UNIT; the host reports %q", strings.TrimSpace(res.Stdout)) + } + } +``` + +with + +```go +func needsTriggerUnit(jobs []app.ScheduledJob) bool { + for _, job := range jobs { + if len(job.Inputs) > 0 { + return true + } + } + return false +} + +// systemdVersion reads the leading number from `systemd 255 (255.4-1ubuntu8)`. +func systemdVersion(firstLine string) (int, bool) { + fields := strings.Fields(firstLine) + if len(fields) < 2 || fields[0] != "systemd" { + return 0, false + } + n, err := strconv.Atoi(fields[1]) + return n, err == nil +} +``` + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine` +Expected: PASS. Existing runner tests that assert `run --rm --no-deps --name 'sample-nightly-1'` must be updated to `run --rm --no-deps "$@" --name 'sample-nightly-1'` (TestScheduledJobUnitContract and TestPinnedScheduledJobLockProtocol). + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine +git commit -m "feat(schedule): manual activations consume declared inputs; hosts need systemd 252 for them + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 10: `Engine.ScheduleRun` and the `schedule_run` operation + +**Files:** +- Create: `internal/engine/schedule_run.go` +- Modify: `internal/onebox/operation_types.go:43,480` (`KindScheduleRun`), `internal/onebox/execution_types.go:288-310,357-373` (`ExecuteRequest.Job/Inputs/Wait`, `OperationResult.ScheduleRun`), `internal/onebox/execute.go:208` (dispatch), `internal/onebox/binding.go:34`, `internal/engine/audit.go:33-57` (`schedule-run` action and outcome) +- Test: `internal/engine/schedule_run_test.go` + +**Interfaces:** +- Produces: + - `type ScheduleRunResult struct { Job, Unit, Operation string; Inputs map[string]string; Started bool; Record *ScheduleRunRecord }` JSON `job, unit, operation, inputs, started, record`. + - `func (e *Engine) ScheduleRun(ctx, operationID, job string, inputs map[string]string, wait bool) (ScheduleRunResult, error)`. + - `KindScheduleRun OperationKind = "schedule_run"`; `ExecuteRequest.Job string`, `ExecuteRequest.Inputs map[string]string`, `ExecuteRequest.Wait bool`; `OperationResult.ScheduleRun *engine.ScheduleRunResult json:"schedule_run,omitempty"`. + - Journal: phase `schedule-run`, `Target: job`, `Detail: "inputs: K=V,..."` (or `inputs: defaults`), events `start` then `finish` (`Status: ok`, `Detail: "unit started; outcome in ob schedule history"`), both before the lock is released; `auditAction("schedule-run") == "schedule run"`, `auditOutcome("schedule run") == "started"`. + +- [ ] **Step 1: Write the failing test** + +```go +package engine + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/transport" +) + +func TestScheduleRunWritesInputsJournalsThenStartsAfterReleasingTheLock(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog", "prices"}, Default: "catalog"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{ExitCode: 3}, true + case strings.Contains(cmd, "systemctl start"): + return transport.Result{}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + result, err := e.ScheduleRun(context.Background(), "20260905-151200-schedule_run-7c1e", "sync", map[string]string{"SOURCE": "prices"}, false) + if err != nil { + t.Fatal(err) + } + if !result.Started || result.Unit != "ob-sample-sync" || result.Inputs["SOURCE"] != "prices" { + t.Fatalf("result = %#v", result) + } + seq := strings.Join(f.Commands, "\n") + inputs := strings.Index(seq, "sync.inputs") + start := strings.Index(seq, "systemctl start --no-block 'ob-sample-sync.service'") + release := strings.LastIndex(seq, "rm -f '/var/lib/ob/sample/lock'") + if inputs < 0 || start < 0 || release < 0 || !(inputs < release && release < start) { + t.Fatalf("expected inputs write, lock release, then start:\n%s", seq) + } + if !strings.Contains(seq, "set -C") { + t.Fatalf("inputs file was not created with noclobber:\n%s", seq) + } + written := strings.Join(f.Inputs, "\n") + for _, want := range []string{"ONEBOX_OPERATION=20260905-151200-schedule_run-7c1e", "SOURCE=prices"} { + if !strings.Contains(written, want) { + t.Fatalf("inputs file is missing %q:\n%s", want, written) + } + } + if !strings.Contains(written, `"phase":"schedule-run"`) || !strings.Contains(written, `"target":"sync"`) { + t.Fatalf("schedule run was not journaled:\n%s", written) + } +} + +func TestScheduleRunRefusals(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog"}, Default: "catalog"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + cfg.Workloads["prune"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "destructive", + Schedule: &app.JobSchedule{Cron: "0 3 * * *", Timezone: "UTC", Timeout: "1h"}, + } + active := false + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl is-active") { + if active { + return transport.Result{Stdout: "activating\n"}, true + } + return transport.Result{ExitCode: 3}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + ctx := context.Background() + if _, err := e.ScheduleRun(ctx, "op", "prune", nil, false); err == nil || !strings.Contains(err.Error(), "ob job run") { + t.Fatalf("destructive job accepted: %v", err) + } + if _, err := e.ScheduleRun(ctx, "op", "sync", map[string]string{"SOURCE": "reviews"}, false); err == nil { + t.Fatal("undeclared value accepted") + } + if _, err := e.ScheduleRun(ctx, "op", "web", nil, false); err == nil { + t.Fatal("non-scheduled workload accepted") + } + active = true + if _, err := e.ScheduleRun(ctx, "op", "sync", nil, false); err == nil || !strings.Contains(err.Error(), "running") { + t.Fatalf("active unit not refused: %v", err) + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/engine -run 'TestScheduleRun'` +Expected: FAIL to compile. + +- [ ] **Step 3: Implement `internal/engine/schedule_run.go`** + +```go +package engine + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/journal" +) + +// ScheduleRunResult is what an operator-initiated run leaves on the +// workstation side. The outcome lives on the host: it is the run record. +type ScheduleRunResult struct { + Job string `json:"job"` + Unit string `json:"unit"` + Operation string `json:"operation"` + Inputs map[string]string `json:"inputs,omitempty"` + Started bool `json:"started"` + Record *ScheduleRunRecord `json:"record,omitempty"` +} + +// ScheduleRun starts a scheduled job's unit now with declared, validated +// inputs. It is journaled like every operation, but the application lock is +// released before the unit starts: the runner exits 75 when it sees that lock. +func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inputs map[string]string, wait bool) (ScheduleRunResult, error) { + result := ScheduleRunResult{Job: name, Operation: operationID, Inputs: inputs} + if strings.TrimSpace(operationID) == "" { + return result, errors.New("schedule run requires an operation id") + } + if err := e.RequireHostOwner(ctx); err != nil { + return result, err + } + job, err := e.scheduledJob(name) + if err != nil { + return result, err + } + workload := e.Spec.Workloads[name] + if workload.DataEffect != app.DataEffectNone { + return result, fmt.Errorf("job %s declares data_effect %q; operator-initiated runs of it go through the sealed plan: ob job plan %s, then ob job run", name, workload.DataEffect, name) + } + if err := app.ValidateJobInputValues(workload, inputs); err != nil { + return result, err + } + unit := e.names().ScheduledJobUnit(name) + result.Unit = unit + + active, err := e.T.Run(ctx, "systemctl is-active "+q(unit+".service")+" 2>/dev/null || true") + if err != nil { + return result, err + } + if state := strings.TrimSpace(active.Stdout); state == "active" || state == "activating" || state == "deactivating" { + return result, fmt.Errorf("job %s is running (%s); wait for it or read ob schedule history %s", name, state, name) + } + + epoch, err := e.AcquireLock(ctx, operationID, e.Opts.ForceLock) + if err != nil { + return result, err + } + locked := true + defer func() { + if locked { + e.ReleaseLock(ctx) + } + }() + if err := e.WriteFence(ctx, operationID, epoch); err != nil { + return result, err + } + + body := scheduleInputsFile(operationID, inputs) + path := e.names().ScheduledJobRunInputs(name) + create := "umask 077 && install -d -m 700 " + q(e.names().AppDir()+"/schedule") + " && set -C && cat > " + q(path) + res, err := e.T.RunInput(ctx, create, body) + if err != nil { + return result, err + } + if res.ExitCode != 0 { + return result, fmt.Errorf("a manual run of %s is already pending (%s exists); wait for it or remove the file on the host", name, path) + } + + writer := &journal.Writer{ + T: e.T, Names: e.names(), DeployID: operationID, Epoch: epoch, Operator: journal.DefaultOperator(), + GitSHA: e.Opts.GitSHA, ConfigHash: e.Opts.ConfigHash, Runner: &e.Opts.Runner, + } + detail := "inputs: defaults" + if len(inputs) > 0 { + detail = "inputs: " + scheduleInputsDetail(inputs) + } + record := journal.Record{Phase: "schedule-run", Event: "start", Status: "ok", Target: name, TargetKind: "job", Detail: detail} + if err := writer.Append(ctx, record); err != nil { + return result, fmt.Errorf("journal schedule run start: %w", err) + } + record.Event, record.Detail = "finish", "unit started; outcome in ob schedule history "+name + if err := writer.Append(ctx, record); err != nil { + return result, fmt.Errorf("journal schedule run finish: %w", err) + } + e.ReleaseLock(ctx) + locked = false + + start := "systemctl start --no-block " + q(unit+".service") + if wait { + start = "systemctl start " + q(unit+".service") + } + res, err = e.mutate(ctx, start) + if err != nil { + return result, err + } + result.Started = true + if res.ExitCode != 0 && !wait { + return result, fmt.Errorf("systemctl start %s: %s", unit, strings.TrimSpace(res.Stderr)) + } + if wait { + records, err := e.ScheduleHistory(ctx, name, 1) + if err != nil { + return result, err + } + if len(records) > 0 { + result.Record = &records[0] + } + if res.ExitCode != 0 && (result.Record == nil || result.Record.Outcome != "skipped") { + return result, fmt.Errorf("job %s did not succeed; see ob schedule logs %s", name, name) + } + } + _ = job + return result, nil +} + +func scheduleInputsFile(operationID string, inputs map[string]string) string { + lines := []string{app.ReservedInputPrefix + "OPERATION=" + operationID} + for _, name := range sortedInputNames(inputs) { + lines = append(lines, name+"="+inputs[name]) + } + return strings.Join(lines, "\n") + "\n" +} + +func scheduleInputsDetail(inputs map[string]string) string { + parts := make([]string, 0, len(inputs)) + for _, name := range sortedInputNames(inputs) { + parts = append(parts, name+"="+inputs[name]) + } + return strings.Join(parts, ",") +} + +func sortedInputNames(inputs map[string]string) []string { + names := make([]string, 0, len(inputs)) + for name := range inputs { + names = append(names, name) + } + sort.Strings(names) + return names +} +``` + +`e.mutate` is fence-checked; if it refuses commands that are not file writes, use `e.T.Run` for the start. Check `internal/engine/lock.go:312` (`mutate`) first and use whichever the existing `systemctl enable --now` path uses (it uses `e.mutate`, so `mutate` is right; but after `ReleaseLock` the fence value is still ours, so it passes). + +Wiring in `internal/onebox`: + +```go +// operation_types.go + KindScheduleRun OperationKind = "schedule_run" +``` + +add `KindScheduleRun` to `validOperationKind` and `operationUsesInspectionRuntime`; add to `ExecuteRequest`: + +```go + // Job, Inputs and Wait are the schedule_run arguments: a declared + // scheduled job, validated input overrides, and whether to block until + // the unit exits. + Job string + Inputs map[string]string + Wait bool +``` + +`OperationResult`: `ScheduleRun *engine.ScheduleRunResult json:"schedule_run,omitempty"`. `execute.go` dispatch: + +```go + case KindScheduleRun: + result.EvidenceID = operationID + var run engine.ScheduleRunResult + run, err = e.ScheduleRun(ctx, operationID, request.Job, request.Inputs, request.Wait) + result.ScheduleRun = &run +``` + +`audit.go`: `case "schedule-run": return "schedule run"` and `case "schedule run": return "started"`. + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/engine ./internal/onebox` +Expected: PASS. If an `internal/onebox` test enumerates every kind (search `validOperationKind` in tests), add `KindScheduleRun` there. + +- [ ] **Step 5: Commit** + +```bash +git add internal/engine/schedule_run.go internal/engine/schedule_run_test.go internal/engine/audit.go internal/onebox +git commit -m "feat(schedule): operator-initiated runs with validated inputs, journaled as schedule_run + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +### Task 11: CLI `ob schedule run` + +**Files:** +- Modify: `cmd/ob/schedule.go`, `cmd/ob/output.go` (matrix), `cmd/ob/output_test.go` (wantMatrix), `site/src/content/docs/reference/policies.mdx` +- Test: `cmd/ob/output_test.go`, `cmd/ob/schedule_test.go` + +**Interfaces:** +- Consumes: `runMutation(cmd, g, onebox.ExecuteRequest{Kind: onebox.KindScheduleRun, Job, Inputs, Wait, BreakLock}, "schedule run")`. +- Produces matrix row `"ob schedule run": {finite_stream, JSON, NDJSON}`; flag parsing `--input NAME=VALUE` (repeatable) into `map[string]string`, refusing a duplicate name or a missing `=` before any connection. + +- [ ] **Step 1: Write the failing tests** + +`cmd/ob/schedule_test.go`: + +```go +package main + +import "testing" + +func TestParseScheduleInputsFlags(t *testing.T) { + got, err := parseScheduleInputs([]string{"SOURCE=prices", "SINCE=2026-09-01"}) + if err != nil || got["SOURCE"] != "prices" || got["SINCE"] != "2026-09-01" { + t.Fatalf("got %#v, %v", got, err) + } + for _, bad := range [][]string{{"SOURCE"}, {"=x"}, {"SOURCE=a", "SOURCE=b"}} { + if _, err := parseScheduleInputs(bad); err == nil { + t.Errorf("%v was accepted", bad) + } + } +} +``` + +Add `"ob schedule run": {Class: "finite_stream", JSON: true, NDJSON: true}` to `wantMatrix`. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./cmd/ob -run 'TestParseScheduleInputsFlags|TestLeafOutputMatrix'` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +```go +func parseScheduleInputs(raw []string) (map[string]string, error) { + out := map[string]string{} + for _, item := range raw { + name, value, ok := strings.Cut(item, "=") + if !ok || name == "" { + return nil, fmt.Errorf("--input %q must be NAME=VALUE", item) + } + if _, dup := out[name]; dup { + return nil, fmt.Errorf("--input %s given twice", name) + } + out[name] = value + } + return out, nil +} +``` + +Command: + +```go + var runInputs []string + var runWait, runBreakLock bool + runCmd := &cobra.Command{ + Use: "run ", + Short: "start a scheduled job now with declared inputs", + Long: "Start one scheduled job's unit now, with values for its declared inputs. Values are validated on the workstation against the declaration; an undeclared name or a value outside its enum or pattern is refused before anything reaches the host.\n\nOnly a job with data_effect none may run this way; a migration or destructive job keeps the sealed plan of ob job run. The request is journaled as schedule_run with the operator and inputs. The outcome is the run record: ob schedule history , or --wait to block until the unit exits and print it.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + inputs, err := parseScheduleInputs(runInputs) + if err != nil { + return writeEarlyOperationFailure(cmd, g, codedError("invalid_argument", "%v", err)) + } + return runMutation(cmd, g, onebox.ExecuteRequest{ + Kind: onebox.KindScheduleRun, Job: args[0], Inputs: inputs, Wait: runWait, BreakLock: runBreakLock, + }, "schedule run") + }, + } + runCmd.Flags().StringArrayVar(&runInputs, "input", nil, "input override as NAME=VALUE; repeatable") + runCmd.Flags().BoolVar(&runWait, "wait", false, "block until the unit exits and print the run record") + runCmd.Flags().BoolVar(&runBreakLock, "break-lock", false, "break a stale operation lock after inspecting its holder") + scheduleCmd.AddCommand(runCmd) +``` + +Check `codedError`'s signature at `cmd/ob/job.go:182` usage and match it. After a successful human-mode run, `runMutation` prints the operation outcome; the record is in `result.ScheduleRun` for structured output. For human output with `--wait`, print the record after `runMutation` returns nil: `result` is not returned by `runMutation`, so add an optional `onResult func(onebox.OperationResult)` hook only if `runMutation` already exposes one; otherwise leave the human hint `run: ob schedule history ` in the command's `Long` and rely on structured output for the record. + +Add `` `ob schedule run` `` to the Finite operation stream row in `policies.mdx`. + +- [ ] **Step 4: Regenerate docs and run tests** + +Run: `just build && just docs-generate && go test ./... && just docs-generate-check && just env-namespace` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add cmd/ob site/src/content/docs/reference +git commit -m "feat(cli): ob schedule run starts a scheduled job now with validated inputs + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +## Docs and end-to-end + +### Task 12: Guide, capabilities page, issue follow-up + +**Files:** +- Modify: `site/src/content/docs/guides/schedule-a-job.mdx`, `site/src/content/docs/status/capabilities.mdx` +- Regenerate: `site/src/content/docs/reference/fields/workloads.mdx`, `site/src/content/docs/reference/cli.mdx`, `site/public/onebox.run-v1.schema.json`, `docs/onebox.run-v1.schema.json` + +- [ ] **Step 1: Guide sections** + +In `schedule-a-job.mdx`: +- Replace the sentence "A failed run is not retried implicitly; the next attempt is the next cron elapse." with a `## Retry inside one firing` section: the `retry` block, attempts counting, doubling capped by `max_backoff`, the timeout bound, what exclusive and pinned hold during backoff, and that the next cron elapse is still the retry for short cadences. +- Rewrite `## Failures remain visible` as `## Every run leaves a record`: the record fields, `ob schedule history `, `ob schedule logs [--run ]`, `ob schedule list`, the new `ob status` line, `skipped` runs, journal retention and the volatile-journal note, and `notify`. +- Add `## Run one now with inputs` before `## Running one by hand`: the `inputs` block, naming and value rules, `ob schedule run --input NAME=VALUE --wait`, the `data_effect: none` rule with the pointer to `ob job run`, the `TRIGGER_UNIT` requirement (systemd 252, Ubuntu 24.04 and Debian 12), and that the request appears in `ob audit` joined to the record by operation id. +- Update the frontmatter `description`/`summary` to mention retry, run history and manual runs. + +- [ ] **Step 2: Capabilities page** + +Under `## Shipped`, add one bullet: + +```markdown +- Scheduled jobs with bounded retry inside one firing, one run record per + activation in the host journal (`ob schedule history`, `ob schedule logs`, + `ob status`), per-outcome notifications, and declared inputs for + operator-initiated runs (`ob schedule run`). +``` + +- [ ] **Step 3: Regenerate and verify** + +Run: `just build && just docs-generate && go run ./cmd/ob schema --out docs/onebox.run-v1.schema.json && just check` +Expected: PASS (`just check` runs mod-tidy, fmt-check, vet, test, docs-generate-check, site-build; `site-build` needs `just site-install` once). + +- [ ] **Step 4: Commit** + +```bash +git add site docs +git commit -m "docs(site): retry, run history, notify and manual runs for scheduled jobs + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +- [ ] **Step 5: Issue follow-up** + +Edit #155: `OB_` becomes `ONEBOX_` (reserved prefix and metadata line), the systemd floor check lives in `SyncSchedules`, the volatile journal is a status note rather than an issue, and `ob schedule run` journals start and finish before releasing the lock. Post one comment summarising the deviations. + +### Task 13: End-to-end coverage on the Lima host + +**Files:** +- Modify: `e2e/testdata/postgres/ob.yml.tmpl:37-51`, `e2e/server_test.go:188-292` + +- [ ] **Step 1: Fixture jobs** + +Add beside `timeout-chore`: + +```yaml + # Fails once, then succeeds: proves the in-firing retry and that the run + # record counts attempts. The marker lives on the host so the second + # attempt, a fresh container, can see the first one ran. + retry-chore: + role: job + image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 + command: ["sh", "-c", "if [ -f /marker/ran ]; then echo second; else touch /marker/ran; exit 1; fi"] + data_effect: none + volumes: [{source: /tmp/onebox-e2e-retry, path: /marker}] + schedule: { cron: "0 0 1 1 *", timeout: 60s, catch_up: false, retry: {attempts: 2, backoff: 1s} } + # A declared input reaches the container as an environment variable, both + # as its default on a timer-shaped start and as an override on a manual run. + input-chore: + role: job + image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 + command: ["sh", "-c", "echo greeting=$GREETING"] + data_effect: none + inputs: + GREETING: {enum: [hi, hello], default: hi} + schedule: { cron: "0 0 1 1 *", timeout: 20s, catch_up: false } +``` + +- [ ] **Step 2: Assertions** + +In the "scheduled jobs" subtest after the normal `chore` run: + +```go + history := s.mustOb(t, dir, "schedule", "history", "chore", "--output", "json") + for _, want := range []string{`"outcome":"success"`, `"trigger":"manual"`, `"attempts":1`} { + if !strings.Contains(history, want) { + t.Fatalf("history is missing %q:\n%s", want, history) + } + } + s.run(t, "rm -rf /tmp/onebox-e2e-retry && mkdir -p /tmp/onebox-e2e-retry") + s.run(t, "systemctl start ob-observer-retry--chore.service") + retry := s.mustOb(t, dir, "schedule", "history", "retry-chore", "--output", "json") + for _, want := range []string{`"outcome":"success"`, `"attempts":2`} { + if !strings.Contains(retry, want) { + t.Fatalf("retry history is missing %q:\n%s", want, retry) + } + } + manual := s.mustOb(t, dir, "schedule", "run", "input-chore", "--input", "GREETING=hello", "--wait", "--output", "json") + if !strings.Contains(manual, `"GREETING":"hello"`) || !strings.Contains(manual, `"outcome":"success"`) { + t.Fatalf("manual run record missing inputs or outcome:\n%s", manual) + } + logs := s.mustOb(t, dir, "schedule", "logs", "input-chore") + if !strings.Contains(logs, "greeting=hello") { + t.Fatalf("run logs do not show the override:\n%s", logs) + } + list := s.mustOb(t, dir, "schedule", "list") + if !strings.Contains(list, "input-chore") || !strings.Contains(list, "active") { + t.Fatalf("schedule list did not show the timer:\n%s", list) + } +``` + +Add to the timeout section: `"last run failed: timeout"` stays; the record for `timeout-chore` shows `"outcome":"timeout"` via `ob schedule history timeout-chore --output json`. + +Check the unit name escaping for `retry-chore` (`ob-observer-retry--chore`, the double hyphen is how `timeout-chore` is spelled at line 192). + +- [ ] **Step 3: Run** + +Run: `command -v limactl && just server-e2e` (boots Ubuntu 24.04, about a minute, then the suite). If Lima is not installed, record that the e2e suite was not run locally and rely on CI. + +- [ ] **Step 4: Commit** + +```bash +git add e2e +git commit -m "test(e2e): scheduled-job retry, run records, manual runs with inputs on a real host + +Claude-Session: https://claude.ai/code/session_013MqwrF5NA179khDtEQdib5" +``` + +## Self-review notes + +- Spec coverage: history (Tasks 1-5), retry and notify (6-7), inputs and manual runs (8-11), docs (12), host validation (9, 13). `ob doctor` is not touched: the journal note lives in `ob status`, and the systemd floor in `SyncSchedules`, both recorded as deviations in Task 12 Step 5. +- Type consistency: `ScheduleRunRecord`, `ScheduleListing`, `ScheduleRunResult`, `JobRetry`, `JobInput`, `KindScheduleRun`, `scheduleRunIdentifier`, `scheduleHistoryCommand`, `scheduleInputsLines`, `scheduleAttemptLoop`, `scheduleNotifier` are used with the same names throughout. diff --git a/e2e/server_test.go b/e2e/server_test.go index 877fee86..86375106 100644 --- a/e2e/server_test.go +++ b/e2e/server_test.go @@ -236,6 +236,46 @@ ExecStart=/usr/bin/docker compose -p observer -f /var/lib/ob/observer/current/co t.Fatalf("normal scheduled run result = %q, want success", result) } + // The notifier wrote a record for that run; a hand-started unit has no + // TRIGGER_UNIT, so it is recorded as a manual activation. + history := s.mustOb(t, dir, "schedule", "history", "chore", "--output", "json") + for _, want := range []string{`"outcome": "success"`, `"trigger": "manual"`, `"attempts": 1`} { + if !strings.Contains(history, want) { + t.Fatalf("run history is missing %q:\n%s", want, history) + } + } + + // One failure, one sleep, one success: the record counts both attempts. + s.run(t, "rm -rf /tmp/onebox-e2e-retry && mkdir -p /tmp/onebox-e2e-retry") + s.run(t, "systemctl start ob-observer-retry--chore.service") + retry := s.mustOb(t, dir, "schedule", "history", "retry-chore", "--output", "json") + for _, want := range []string{`"outcome": "success"`, `"attempts": 2`} { + if !strings.Contains(retry, want) { + t.Fatalf("retry history is missing %q:\n%s", want, retry) + } + } + + // A manual run with an input override reaches the container as its + // environment, is journaled with the operator, and shows up as such. + manual := s.mustOb(t, dir, "schedule", "run", "input-chore", "--input", "GREETING=hello", "--wait", "--output", "json") + for _, want := range []string{`"GREETING": "hello"`, `"outcome": "success"`, `"trigger": "manual"`} { + if !strings.Contains(manual, want) { + t.Fatalf("manual run result is missing %q:\n%s", want, manual) + } + } + logs := s.mustOb(t, dir, "schedule", "logs", "input-chore") + if !strings.Contains(logs, "greeting=hello") { + t.Fatalf("run logs do not show the override:\n%s", logs) + } + audit := s.mustOb(t, dir, "audit") + if !strings.Contains(audit, "schedule run") { + t.Fatalf("audit does not list the manual run:\n%s", audit) + } + list := s.mustOb(t, dir, "schedule", "list") + if !strings.Contains(list, "input-chore") || !strings.Contains(list, "active") { + t.Fatalf("schedule list did not show the timer:\n%s", list) + } + // The receiver lives on the target because host-fired notifications do // too. It accepts one POST, records the body, and exits. receiver := `from http.server import BaseHTTPRequestHandler, HTTPServer @@ -271,6 +311,10 @@ HTTPServer(("127.0.0.1", 18080), Handler).handle_request() "systemctl show ob-observer-timeout--chore.service --property=Result --value")); result != "timeout" { t.Fatalf("timed-out scheduled run result = %q, want timeout", result) } + // The runner was killed mid-run; ExecStopPost still wrote the record. + if timedOut := s.mustOb(t, dir, "schedule", "history", "timeout-chore", "--output", "json"); !strings.Contains(timedOut, `"outcome": "timeout"`) { + t.Fatalf("timeout was not recorded:\n%s", timedOut) + } out, err := s.ob(t, dir, "status") if err == nil || !strings.Contains(out, "schedule timeout-chore") || !strings.Contains(out, "last run failed: timeout") { t.Fatalf("status did not expose the scheduled failure (err=%v):\n%s", err, out) @@ -286,9 +330,14 @@ HTTPServer(("127.0.0.1", 18080), Handler).handle_request() t.Fatalf("scheduled failure notification is missing %q: %s", want, notification) } } - // Do not make the deliberately induced failure pollute later lifecycle - // assertions; systemd resets Result to success with the failed state. - s.run(t, "systemctl reset-failed ob-observer-timeout--chore.service") + // The record is the verdict, so `systemctl reset-failed` no longer + // clears the failure from `ob status`; only a later successful run + // does. A manual run with the input that makes the job finish in time + // is that run, and it must leave status green for the steps after. + s.mustOb(t, dir, "schedule", "run", "timeout-chore", "--input", "SLEEP=0", "--wait") + if cleared := s.mustOb(t, dir, "status"); !strings.Contains(cleared, "schedule timeout-chore active") { + t.Fatalf("a successful manual run did not clear the recorded timeout:\n%s", cleared) + } }) t.Run("preflight", func(t *testing.T) { diff --git a/e2e/testdata/postgres/ob.yml.tmpl b/e2e/testdata/postgres/ob.yml.tmpl index f22ba03f..000d8345 100644 --- a/e2e/testdata/postgres/ob.yml.tmpl +++ b/e2e/testdata/postgres/ob.yml.tmpl @@ -41,14 +41,38 @@ workloads: data_effect: none schedule: { cron: "0 0 1 1 *", timeout: 20s, catch_up: false } # A deliberately wedged timer run proves the host-enforced timeout becomes a - # failed systemd result and that `ob status` exposes it. Its annual timer never - # fires during the suite; the test starts the service directly. + # recorded timeout that `ob status` exposes. Its annual timer never fires + # during the suite; the test starts the service directly. The SLEEP input + # lets a later manual run succeed, which is the only thing that clears a + # recorded failure. timeout-chore: role: job image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 - command: ["sh", "-c", "sleep 30"] + command: ["sh", "-c", "sleep $SLEEP"] data_effect: none + inputs: + SLEEP: {enum: ["0", "30"], default: "30"} schedule: { cron: "0 0 1 1 *", timeout: 1s, catch_up: false } + # Fails once, then succeeds: proves the in-firing retry and that the run + # record counts attempts. The marker lives on the host so the second + # attempt, a fresh container, can see the first one ran. + retry-chore: + role: job + image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 + command: ["sh", "-c", "if [ -f /marker/ran ]; then echo second; else touch /marker/ran; exit 1; fi"] + data_effect: none + volumes: [{source: /tmp/onebox-e2e-retry, path: /marker}] + schedule: { cron: "0 0 1 1 *", timeout: 60s, catch_up: false, retry: {attempts: 2, backoff: 1s} } + # A declared input reaches the container as an environment variable: its + # default on a timer-shaped start, an override on a manual run. + input-chore: + role: job + image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 + command: ["sh", "-c", "echo greeting=$GREETING"] + data_effect: none + inputs: + GREETING: {enum: [hi, hello], default: hi} + schedule: { cron: "0 0 1 1 *", timeout: 20s, catch_up: false } deployment: order: [app] services: diff --git a/internal/app/constraints.go b/internal/app/constraints.go index 87429961..bd75732f 100644 --- a/internal/app/constraints.go +++ b/internal/app/constraints.go @@ -171,6 +171,7 @@ var ( eMigrationPolicy = []string{"manual", "auto", "expand-only"} eNotifyFormat = []string{"text", "json"} eNotifyEvent = []string{"success", "failure"} + eScheduleNotify = []string{"success", "failure", "timeout", "skipped"} eProxyKind = []string{"traefik-docker", "none"} // One provider, because one is implemented. The withdrawn `secrets` block // accepted `age` and nothing ever decrypted it — every path shells out to diff --git a/internal/app/generate.go b/internal/app/generate.go index 2087c781..1ba9e7a9 100644 --- a/internal/app/generate.go +++ b/internal/app/generate.go @@ -344,7 +344,19 @@ func (p *Spec) renderWorkload(n Names, name string, w Workload, releaseID string } svc["labels"] = labels - if env := stringMap(w.Env); len(env) > 0 { + env := stringMap(w.Env) + // A declared input's default is part of the release, so a timer firing, a + // manual run without overrides, and a hand-typed `docker compose run` all + // see the same value. Validation refuses a name that is also an env key. + if len(w.Inputs) > 0 { + if env == nil { + env = map[string]any{} + } + for name, in := range w.Inputs { + env[name] = in.Default + } + } + if len(env) > 0 { svc["environment"] = env } // A workload that needs a service reads how to reach it. The file is diff --git a/internal/app/jsonschema.go b/internal/app/jsonschema.go index ebc55a67..5a360278 100644 --- a/internal/app/jsonschema.go +++ b/internal/app/jsonschema.go @@ -129,7 +129,8 @@ func annotateSchemaField(schema map[string]any, field reflect.StructField) { } func schemaTagValue(value string, t reflect.Type) any { - switch deref(t).Kind() { + target := deref(t) + switch target.Kind() { case reflect.Bool: if parsed, err := strconv.ParseBool(value); err == nil { return parsed @@ -142,6 +143,22 @@ func schemaTagValue(value string, t reflect.Type) any { if parsed, err := strconv.ParseFloat(value, 64); err == nil { return parsed } + case reflect.Slice: + // A list's default is written in the tag the way it reads in prose, + // `success, failure`, because that is what the field reference prints. + // The schema needs the value itself: a string default on an array + // property is a contradiction, and an editor that applies defaults + // would fill the list with one sentence. + parts := strings.Split(value, ",") + out := make([]any, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + out = append(out, schemaTagValue(part, target.Elem())) + } + return out } return value } @@ -343,6 +360,17 @@ var schemaConstraints = []struct { {[]string{"workloads", "*", "drain", "signal"}, pattern(gSignal)}, {[]string{"workloads", "*", "drain", "wait"}, pattern(gDur)}, {[]string{"workloads", "*", "drain", "grace"}, pattern(gDur)}, + {[]string{"workloads", "*", "schedule", "notify", "items"}, enum(eScheduleNotify)}, + {[]string{"workloads", "*", "schedule", "retry", "attempts"}, map[string]any{"minimum": 1, "maximum": maxRetryAttempts}}, + {[]string{"workloads", "*", "schedule", "retry", "backoff"}, pattern(gDur)}, + {[]string{"workloads", "*", "schedule", "retry", "max_backoff"}, pattern(gDur)}, + {[]string{"workloads", "*", "inputs"}, propertyNames(gInputName)}, + // An input says what it accepts, one way, and always has a default: the + // timer fires without anyone to ask. + {[]string{"workloads", "*", "inputs", "*"}, map[string]any{ + "required": []any{"default"}, + "oneOf": anyRequired([]any{"enum", "pattern"}), + }}, {[]string{"workloads", "*", "resources", "memory"}, pattern(gSize)}, {[]string{"workloads", "*", "resources", "cpus"}, pattern(gCpus)}, {[]string{"workloads", "*", "persistence", "mode"}, enum(ePersistence)}, @@ -494,7 +522,7 @@ func applyRoleRules(doc map[string]any) { map[string]any{"anyOf": anyRequired(sources)}, } - jobOnly := []any{"when", "data_effect", "schedule"} + jobOnly := []any{"when", "data_effect", "schedule", "inputs"} workload["allOf"] = []any{ // Exactly one source. A workload with none cannot run and a workload // with two does not say which image it is. diff --git a/internal/app/jsonschema_test.go b/internal/app/jsonschema_test.go index 34f3c764..3c74dd64 100644 --- a/internal/app/jsonschema_test.go +++ b/internal/app/jsonschema_test.go @@ -3,6 +3,7 @@ package app import ( "bytes" "encoding/json" + "fmt" "os" "path/filepath" "reflect" @@ -309,3 +310,62 @@ func TestEveryJSONSchemaExemptionNamesARealCase(t *testing.T) { } } } + +// A JSON Schema default has to be a value of the property's own type. A list +// whose default is a sentence is a contradiction, and an editor that applies +// defaults would fill the list with that sentence. +func TestPublishedSchemaGivesListFieldsListDefaultsAndExamples(t *testing.T) { + body, err := JSONSchema() + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatal(err) + } + var walk func(node map[string]any, path string) + walk = func(node map[string]any, path string) { + if node["type"] == "array" { + if value, ok := node["default"]; ok { + if _, isList := value.([]any); !isList { + t.Errorf("%s is an array whose default is %T (%v)", path, value, value) + } + } + for i, example := range asList(node["examples"]) { + if _, isList := example.([]any); !isList { + t.Errorf("%s example %d is %T (%v), not a value of the array type", path, i, example, example) + } + } + } + for _, key := range []string{"properties", "patternProperties"} { + for name, child := range asMap(node[key]) { + if sub, ok := child.(map[string]any); ok { + walk(sub, path+"."+name) + } + } + } + for _, key := range []string{"items", "additionalProperties"} { + if sub, ok := node[key].(map[string]any); ok { + walk(sub, path+"."+key) + } + } + for _, key := range []string{"allOf", "anyOf", "oneOf"} { + for i, branch := range asList(node[key]) { + if sub, ok := branch.(map[string]any); ok { + walk(sub, fmt.Sprintf("%s.%s[%d]", path, key, i)) + } + } + } + } + walk(doc, "") +} + +func asList(v any) []any { + out, _ := v.([]any) + return out +} + +func asMap(v any) map[string]any { + out, _ := v.(map[string]any) + return out +} diff --git a/internal/app/names.go b/internal/app/names.go index 5f36a648..6891c123 100644 --- a/internal/app/names.go +++ b/internal/app/names.go @@ -244,6 +244,12 @@ func (n Names) ScheduledJobRunState(job string) string { return path.Join(n.AppDir(), "schedule", job+".state") } +// ScheduledJobRunInputs is the one-shot file `ob schedule run` leaves for the +// next manual activation. The runner consumes and deletes it. +func (n Names) ScheduledJobRunInputs(job string) string { + return path.Join(n.AppDir(), "schedule", job+".inputs") +} + // ScheduledJobUnitPrefixes returns the current namespace followed by the // pre-2026.8.6 spelling when the application name contains a hyphen. func (n Names) ScheduledJobUnitPrefixes() []string { diff --git a/internal/app/schedule.go b/internal/app/schedule.go index 531f1bd4..f74e0d93 100644 --- a/internal/app/schedule.go +++ b/internal/app/schedule.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" "strings" + "time" ) // A scheduled job is a promise that something runs at a time nobody will be @@ -32,6 +33,14 @@ type ScheduledJob struct { DeployLock string // Calendar is the host-side expression the cron translates to. Calendar string + // Retry and notify policy, resolved over the defaults so the runner and + // notifier renderers never see an absent block. + RetryAttempts int + RetryBackoff time.Duration + RetryMaxBackoff time.Duration + Notify []string + // Inputs are the declared parameters a manual run may override. + Inputs map[string]JobInput } // ScheduledJobs lists every job with a schedule, in a stable order. @@ -55,9 +64,12 @@ func (p *Spec) ScheduledJobs() ([]ScheduledJob, error) { if deployLock == "" { deployLock = "exclusive" } + attempts, backoff, maxBackoff := w.Schedule.retryPolicy() out = append(out, ScheduledJob{ Name: name, Cron: w.Schedule.Cron, Timezone: tz, Calendar: cal, Timeout: w.Schedule.Timeout, CatchUp: w.Schedule.CatchUp, DeployLock: deployLock, + RetryAttempts: attempts, RetryBackoff: backoff, RetryMaxBackoff: maxBackoff, + Notify: w.Schedule.notifyOutcomes(), Inputs: w.Inputs, }) } return out, nil diff --git a/internal/app/schedule_inputs.go b/internal/app/schedule_inputs.go new file mode 100644 index 00000000..90bb0c35 --- /dev/null +++ b/internal/app/schedule_inputs.go @@ -0,0 +1,123 @@ +package app + +import ( + "fmt" + "regexp" + "strings" +) + +// ReservedInputPrefix is Onebox's environment namespace. Declared inputs may +// not enter it, and the manual-run metadata line the runner reads lives there. +const ReservedInputPrefix = "ONEBOX_" + +// maxInputValueBytes bounds a value the way the exec reason is bounded: it is +// public operational metadata that travels through a file, a command line and +// a journal record, and none of those want a paragraph. +const maxInputValueBytes = 256 + +// gInputName is deliberately narrower than gEnvName: inputs become +// environment variables, and the upper-case convention keeps them visibly +// distinct from whatever the image sets for itself. +var gInputName = grammar{"input name", regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`), + "upper-case letters, digits and underscores, starting with a letter"} + +// InputValueAllowed is the charset that lets the runner pass a value through +// to the container and into the run record without escaping anything. It is +// checked regardless of the declared pattern, because a pattern is the +// project's promise about meaning, not the runner's guarantee about shell. +func InputValueAllowed(v string) bool { + if len(v) > maxInputValueBytes { + return false + } + for _, r := range v { + if r == '"' || r == '\\' || r < 0x20 || r == 0x7f { + return false + } + } + return true +} + +// accepts reports whether v satisfies this input. A pattern matches the whole +// value: an author who writes `[0-9]+` means a number, not a value that +// contains one somewhere. +func (in JobInput) accepts(v string) bool { + if !InputValueAllowed(v) { + return false + } + if len(in.Enum) > 0 { + for _, allowed := range in.Enum { + if v == allowed { + return true + } + } + return false + } + re, err := regexp.Compile("^(?:" + in.Pattern + ")$") + if err != nil { + return false + } + return re.MatchString(v) +} + +// ValidateJobInputValues checks operator-supplied overrides against the +// declaration. Defaults were checked at load; this is the other half, run on +// the workstation before anything reaches the host. +func ValidateJobInputValues(w Workload, values map[string]string) error { + for _, name := range sortedKeys(values) { + in, ok := w.Inputs[name] + if !ok { + return fmt.Errorf("input %s is not declared by this job", name) + } + if !in.accepts(values[name]) { + return fmt.Errorf("input %s: %q is not an accepted value", name, values[name]) + } + } + return nil +} + +func validateJobInputs(w Workload, path string) error { + if len(w.Inputs) == 0 { + return nil + } + if w.Schedule == nil { + return errf("project_invalid", path+".inputs", "", + "inputs belong to a scheduled job; declare schedule or remove inputs") + } + if w.DataEffect != DataEffectNone { + return errf("project_invalid", path+".inputs", "", + "inputs require data_effect %q; a %q job keeps the sealed plan of ob job run for operator-initiated runs", + DataEffectNone, w.DataEffect) + } + for _, name := range sortedKeys(w.Inputs) { + ip := path + ".inputs." + name + if err := gInputName.check(ip, name); err != nil { + return err + } + if strings.HasPrefix(name, ReservedInputPrefix) { + return errf("project_invalid", ip, "", "%s is Onebox's environment namespace; choose another name", ReservedInputPrefix) + } + if _, clash := w.Env[name]; clash { + return errf("project_invalid", ip, "", "input %s collides with an env key of the same name", name) + } + in := w.Inputs[name] + if (len(in.Enum) > 0) == (in.Pattern != "") { + return errf("project_invalid", ip, "", "declare exactly one of enum or pattern") + } + if in.Pattern != "" { + if _, err := regexp.Compile("^(?:" + in.Pattern + ")$"); err != nil { + return errf("project_invalid", ip+".pattern", "", "%v", err) + } + } + for i, v := range in.Enum { + if !InputValueAllowed(v) { + return errf("project_invalid", indexed(ip+".enum", i), "", + "enum values may not contain quotes, backslashes or control characters and are at most %d bytes", maxInputValueBytes) + } + } + if !in.accepts(in.Default) { + return errf("project_invalid", ip+".default", "", + "default %q does not satisfy the input's own constraint", in.Default) + } + } + return nil +} diff --git a/internal/app/schedule_retry.go b/internal/app/schedule_retry.go new file mode 100644 index 00000000..2c728b89 --- /dev/null +++ b/internal/app/schedule_retry.go @@ -0,0 +1,116 @@ +package app + +import ( + "math" + "time" +) + +// The retry defaults reproduce today's behaviour exactly: one attempt, and the +// backoff values are inert until attempts rises above one. +const ( + defaultRetryAttempts = 1 + defaultRetryBackoff = 30 * time.Second + defaultRetryMaxBackoff = 10 * time.Minute + maxRetryAttempts = 10 +) + +// defaultScheduleNotify is what the failure notifier always did: speak on +// failure and timeout, stay quiet on success, and never mention a skip. +var defaultScheduleNotify = []string{"failure", "timeout"} + +// RetryBackoffSeconds is the whole-second form the runner sleeps: `sleep` +// takes seconds, and a fraction rounds up rather than down to a busy loop. +func RetryBackoffSeconds(d time.Duration) int { + return int(math.Ceil(d.Seconds())) +} + +// scheduleRetryWorstCase is the longest a run can spend asleep between +// attempts. It reproduces the runner's own arithmetic step for step: whole +// seconds, sleep, double, cap, over the attempts-1 sleeps. Validation keeps it +// under the timeout so the last attempt can always start, and that promise +// only holds if both sides count the same way. +func scheduleRetryWorstCase(attempts int, backoff, max time.Duration) time.Duration { + total := 0 + sleep, cap := RetryBackoffSeconds(backoff), RetryBackoffSeconds(max) + for i := 1; i < attempts; i++ { + total += sleep + sleep *= 2 + if sleep > cap { + sleep = cap + } + } + return time.Duration(total) * time.Second +} + +// retryPolicy resolves the declared block over the defaults. Unparseable +// durations fall back to the default here; validation has already refused them +// on the load path, so this only softens a struct built by hand. +func (s *JobSchedule) retryPolicy() (attempts int, backoff, max time.Duration) { + attempts, backoff, max = defaultRetryAttempts, defaultRetryBackoff, defaultRetryMaxBackoff + if s == nil || s.Retry == nil { + return attempts, backoff, max + } + if s.Retry.Attempts != nil { + attempts = *s.Retry.Attempts + } + if s.Retry.Backoff != "" { + if d, ok := ParseDuration(s.Retry.Backoff); ok { + backoff = d + } + } + if s.Retry.MaxBackoff != "" { + if d, ok := ParseDuration(s.Retry.MaxBackoff); ok { + max = d + } + } + return attempts, backoff, max +} + +// notifyOutcomes resolves the declared list over the default. +func (s *JobSchedule) notifyOutcomes() []string { + if s == nil || len(s.Notify) == 0 { + return append([]string(nil), defaultScheduleNotify...) + } + return append([]string(nil), s.Notify...) +} + +// scheduleTimeout is the run's wall-time bound as a duration, with the +// schema default when the author left it out. +func (s *JobSchedule) scheduleTimeout() time.Duration { + if s != nil && s.Timeout != "" { + if d, ok := ParseDuration(s.Timeout); ok { + return d + } + } + return time.Hour +} + +func validateJobRetry(s *JobSchedule, path string) error { + if s.Retry != nil { + if s.Retry.Attempts != nil && (*s.Retry.Attempts < 1 || *s.Retry.Attempts > maxRetryAttempts) { + return errf("project_invalid", path+".retry.attempts", "", + "attempts must be between 1 and %d, got %d", maxRetryAttempts, *s.Retry.Attempts) + } + if err := gDur.checkOptional(path+".retry.backoff", s.Retry.Backoff); err != nil { + return err + } + if err := gDur.checkOptional(path+".retry.max_backoff", s.Retry.MaxBackoff); err != nil { + return err + } + attempts, backoff, max := s.retryPolicy() + if backoff > max { + return errf("project_invalid", path+".retry.backoff", "", + "backoff %s exceeds max_backoff %s", backoff, max) + } + if worst, timeout := scheduleRetryWorstCase(attempts, backoff, max), s.scheduleTimeout(); worst >= timeout { + return errf("project_invalid", path+".retry", "", + "worst-case backoff %s is not smaller than timeout %s; later attempts could never start", worst, timeout) + } + } + for i, outcome := range s.Notify { + if err := checkEnum(indexed(path+".notify", i), outcome, eScheduleNotify); err != nil { + return err + } + } + return nil +} diff --git a/internal/app/schedule_test.go b/internal/app/schedule_test.go index 72363233..3dc250fa 100644 --- a/internal/app/schedule_test.go +++ b/internal/app/schedule_test.go @@ -1,8 +1,11 @@ package app import ( + "errors" + "fmt" "strings" "testing" + "time" ) // A schedule that silently never fires looks exactly like one that works, @@ -148,3 +151,200 @@ workloads: t.Fatal("an invalid scheduled-job timeout was accepted") } } + +func TestScheduledJobRetryAndNotifyResolveWithDefaults(t *testing.T) { + spec, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + plain: + role: job + image: x:1 + data_effect: none + schedule: {cron: "0 3 * * *"} + retrying: + role: job + image: x:1 + data_effect: none + schedule: + cron: "0 * * * *" + timeout: 45m + retry: {attempts: 3, backoff: 30s, max_backoff: 10m} + notify: [failure, timeout, skipped] +`), "ob.yml") + if err != nil { + t.Fatal(err) + } + jobs, err := spec.ScheduledJobs() + if err != nil { + t.Fatal(err) + } + byName := map[string]ScheduledJob{} + for _, job := range jobs { + byName[job.Name] = job + } + plain := byName["plain"] + if plain.RetryAttempts != 1 || plain.RetryBackoff != 30*time.Second || plain.RetryMaxBackoff != 10*time.Minute || + strings.Join(plain.Notify, ",") != "failure,timeout" { + t.Fatalf("defaults did not resolve: %#v", plain) + } + retrying := byName["retrying"] + if retrying.RetryAttempts != 3 || strings.Join(retrying.Notify, ",") != "failure,timeout,skipped" { + t.Fatalf("declared retry did not resolve: %#v", retrying) + } +} + +func TestScheduledJobRetryIsBoundedByTheTimeout(t *testing.T) { + for name, tc := range map[string]struct { + schedule string + code string + }{ + "too many attempts": {`{cron: "0 * * * *", retry: {attempts: 11}}`, "project_invalid"}, + "zero attempts": {`{cron: "0 * * * *", retry: {attempts: 0}}`, "project_invalid"}, + "backoff over max": {`{cron: "0 * * * *", retry: {attempts: 2, backoff: 20m, max_backoff: 10m}}`, "project_invalid"}, + "backoff exceeds timeout": {`{cron: "0 * * * *", timeout: 5m, retry: {attempts: 3, backoff: 2m, max_backoff: 30m}}`, "project_invalid"}, + "unknown notify": {`{cron: "0 * * * *", notify: [warning]}`, "project_invalid"}, + } { + t.Run(name, func(t *testing.T) { + _, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + j: {role: job, image: x:1, data_effect: none, schedule: `+tc.schedule+`} +`), "ob.yml") + var e *Error + if !errors.As(err, &e) || e.Code != tc.code { + t.Fatalf("err = %v, want code %s", err, tc.code) + } + }) + } + if got := scheduleRetryWorstCase(3, 2*time.Minute, 30*time.Minute); got != 6*time.Minute { + t.Fatalf("worst case = %s, want 6m", got) + } + if got := scheduleRetryWorstCase(4, 30*time.Second, time.Minute); got != 150*time.Second { + t.Fatalf("capped worst case = %s, want 2m30s", got) + } +} + +func TestJobInputsValidateNamesConstraintsAndDefaults(t *testing.T) { + base := `api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + sync: + role: job + image: x:1 + data_effect: %s + env: {MODE: fast} + %s + inputs: + %s +` + load := func(effect, schedule, inputs string) error { + _, err := LoadBytes([]byte(fmt.Sprintf(base, effect, schedule, inputs)), "ob.yml") + return err + } + good := "SOURCE: {enum: [catalog, prices], default: catalog, description: Which upstream.}" + if err := load("none", `schedule: {cron: "0 * * * *"}`, good); err != nil { + t.Fatalf("valid inputs refused: %v", err) + } + for name, tc := range map[string]struct{ effect, schedule, inputs string }{ + "no schedule": {"none", "", good}, + "destructive job": {"destructive", `schedule: {cron: "0 * * * *"}`, good}, + "lowercase name": {"none", `schedule: {cron: "0 * * * *"}`, "source: {enum: [a], default: a}"}, + "reserved prefix": {"none", `schedule: {cron: "0 * * * *"}`, "ONEBOX_X: {enum: [a], default: a}"}, + "collides with env": {"none", `schedule: {cron: "0 * * * *"}`, "MODE: {enum: [a], default: a}"}, + "enum and pattern": {"none", `schedule: {cron: "0 * * * *"}`, "S: {enum: [a], pattern: '^a$', default: a}"}, + "neither": {"none", `schedule: {cron: "0 * * * *"}`, "S: {default: a}"}, + "default off enum": {"none", `schedule: {cron: "0 * * * *"}`, "S: {enum: [a], default: b}"}, + "default off pattern": {"none", `schedule: {cron: "0 * * * *"}`, "S: {pattern: '^[0-9]+$', default: x}"}, + "quote in default": {"none", `schedule: {cron: "0 * * * *"}`, `S: {pattern: '.*', default: 'a"b'}`}, + "bad regex": {"none", `schedule: {cron: "0 * * * *"}`, "S: {pattern: '(', default: a}"}, + } { + t.Run(name, func(t *testing.T) { + var e *Error + if err := load(tc.effect, tc.schedule, tc.inputs); !errors.As(err, &e) || e.Code != "project_invalid" { + t.Fatalf("err = %v, want project_invalid", err) + } + }) + } + if _, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + web: {role: application, image: x:1, inputs: {S: {enum: [a], default: a}}} +`), "ob.yml"); err == nil { + t.Fatal("inputs on a non-job workload were accepted") + } +} + +func TestValidateJobInputValuesChecksOverrides(t *testing.T) { + w := Workload{Role: RoleJob, Inputs: map[string]JobInput{ + "SOURCE": {Enum: []string{"catalog", "prices"}, Default: "catalog"}, + "SINCE": {Pattern: `^([0-9]{4}-[0-9]{2}-[0-9]{2})?$`, Default: ""}, + }} + if err := ValidateJobInputValues(w, map[string]string{"SOURCE": "prices", "SINCE": "2026-09-01"}); err != nil { + t.Fatal(err) + } + if err := ValidateJobInputValues(w, nil); err != nil { + t.Fatal(err) + } + for name, values := range map[string]map[string]string{ + "unknown": {"OTHER": "x"}, + "off enum": {"SOURCE": "reviews"}, + "off pattern": {"SINCE": "yesterday"}, + "backslash": {"SINCE": `2026\-09-01`}, + "newline": {"SOURCE": "prices\n"}, + "partial match": {"SINCE": "x2026-09-01"}, + } { + if err := ValidateJobInputValues(w, values); err == nil { + t.Errorf("%s was accepted", name) + } + } + if InputValueAllowed(strings.Repeat("a", 257)) { + t.Error("an oversized value was allowed") + } + if !InputValueAllowed("a value with spaces, commas, and unicode ✓") { + t.Error("an ordinary value was refused") + } +} + +func TestScheduledJobInputDefaultsRenderIntoTheComposeEnvironment(t *testing.T) { + spec, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + sync: + role: job + image: x:1 + data_effect: none + env: {MODE: fast} + schedule: {cron: "0 * * * *"} + inputs: + SOURCE: {enum: [catalog, prices], default: catalog} +`), "ob.yml") + if err != nil { + t.Fatal(err) + } + r, err := spec.Resolve("production") + if err != nil { + t.Fatal(err) + } + out, err := r.Render("production", "R1", nil) + if err != nil { + t.Fatal(err) + } + rendered := string(out.Bytes) + for _, want := range []string{"SOURCE: catalog", "MODE: fast"} { + if !strings.Contains(rendered, want) { + t.Fatalf("rendered runtime is missing %q:\n%s", want, rendered) + } + } + jobs, err := spec.ScheduledJobs() + if err != nil { + t.Fatal(err) + } + if jobs[0].Inputs["SOURCE"].Default != "catalog" { + t.Fatalf("scheduled job did not carry its inputs: %#v", jobs[0]) + } +} diff --git a/internal/app/types.go b/internal/app/types.go index 598146f3..841a3247 100644 --- a/internal/app/types.go +++ b/internal/app/types.go @@ -168,9 +168,21 @@ type Workload struct { Logging *Logging `json:"logging,omitempty" description:"Container logging driver and driver-specific options."` // Job only. - When string `json:"when,omitempty" description:"When a job runs: manual, pre_release, or post_release." default:"manual"` - DataEffect DataEffect `json:"data_effect,omitempty" description:"Job data impact used by rollback and abort gates." example:"migration"` - Schedule *JobSchedule `json:"schedule,omitempty" description:"Host-resident recurring schedule and run policy for a job."` + When string `json:"when,omitempty" description:"When a job runs: manual, pre_release, or post_release." default:"manual"` + DataEffect DataEffect `json:"data_effect,omitempty" description:"Job data impact used by rollback and abort gates." example:"migration"` + Schedule *JobSchedule `json:"schedule,omitempty" description:"Host-resident recurring schedule and run policy for a job."` + Inputs map[string]JobInput `json:"inputs,omitempty" description:"Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them."` +} + +// JobInput is one declared parameter of a scheduled job. The constraint is +// what makes a manual run safe to accept from a command line: a value is +// either one of the listed words or matches the pattern, and never contains a +// character the runner would have to escape. +type JobInput struct { + Enum []string `json:"enum,omitempty" description:"Accepted values." example:"catalog"` + Pattern string `json:"pattern,omitempty" description:"Regular expression the whole value must match." example:"^[0-9]{4}-[0-9]{2}-[0-9]{2}$"` + Default string `json:"default" description:"Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint."` + Description string `json:"description,omitempty" description:"What the input controls."` } type Build struct { @@ -266,11 +278,22 @@ type Schedule struct { } type JobSchedule struct { - Cron string `json:"cron" description:"Five-field cron schedule translated to a host timer." example:"0 2 * * *"` - Timezone string `json:"timezone" description:"IANA timezone used to interpret the cron schedule." default:"UTC" example:"Europe/Berlin"` - Timeout string `json:"timeout" description:"Maximum wall time for one scheduled run before systemd terminates it and records failure." default:"1h" example:"30m"` - CatchUp bool `json:"catch_up" description:"Run once after the host returns if an elapsed schedule was missed while it was offline." default:"true"` - DeployLock string `json:"deploy_lock" description:"Deployment coordination policy: exclusive blocks application operations for the full run; pinned leases the immutable starting release and permits only deployments without data-changing jobs or untyped hooks." default:"exclusive" example:"pinned"` + Cron string `json:"cron" description:"Five-field cron schedule translated to a host timer." example:"0 2 * * *"` + Timezone string `json:"timezone" description:"IANA timezone used to interpret the cron schedule." default:"UTC" example:"Europe/Berlin"` + Timeout string `json:"timeout" description:"Maximum wall time for one scheduled run before systemd terminates it and records failure." default:"1h" example:"30m"` + CatchUp bool `json:"catch_up" description:"Run once after the host returns if an elapsed schedule was missed while it was offline." default:"true"` + DeployLock string `json:"deploy_lock" description:"Deployment coordination policy: exclusive blocks application operations for the full run; pinned leases the immutable starting release and permits only deployments without data-changing jobs or untyped hooks." default:"exclusive" example:"pinned"` + Retry *JobRetry `json:"retry,omitempty" description:"Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run."` + Notify []string `json:"notify,omitempty" description:"Run outcomes that send the configured notifications: success, failure, timeout, skipped." default:"failure, timeout"` +} + +// JobRetry bounds how a scheduled run recovers from a transient failure. The +// sleeps happen under the locks the run already holds, so validation keeps +// their worst-case sum under the schedule's timeout. +type JobRetry struct { + Attempts *int `json:"attempts,omitempty" description:"Total attempts including the first, 1 to 10." default:"1" example:"3"` + Backoff string `json:"backoff,omitempty" description:"Sleep before the second attempt; it doubles after each failure." default:"30s" example:"1m"` + MaxBackoff string `json:"max_backoff,omitempty" description:"Upper bound for the doubling sleep." default:"10m" example:"30m"` } type Service struct { diff --git a/internal/app/validate.go b/internal/app/validate.go index c1773dfe..3af3d6f1 100644 --- a/internal/app/validate.go +++ b/internal/app/validate.go @@ -455,9 +455,12 @@ func validateWorkload(w Workload, path string) error { "pinned scheduled runs require a Onebox-rendered workload; adopted Compose may reference files outside the leased release") } } - } else if w.When != "" || w.DataEffect != "" || w.Schedule != nil { + if err := validateJobInputs(w, path); err != nil { + return err + } + } else if w.When != "" || w.DataEffect != "" || w.Schedule != nil || len(w.Inputs) > 0 { return errf("project_invalid", path, "", - "when, data_effect and schedule belong to a job; this workload's role is %q", w.Role) + "when, data_effect, schedule and inputs belong to a job; this workload's role is %q", w.Role) } return nil } @@ -580,6 +583,9 @@ func validateJobSchedule(s *JobSchedule, path string) error { if err := gDur.check(path+".timeout", s.Timeout); err != nil { return err } + if err := validateJobRetry(s, path); err != nil { + return err + } return checkEnum(path+".deploy_lock", s.DeployLock, eScheduleDeployLock) } diff --git a/internal/engine/audit.go b/internal/engine/audit.go index 6537b2c2..2593486d 100644 --- a/internal/engine/audit.go +++ b/internal/engine/audit.go @@ -173,6 +173,8 @@ func auditAction(phase string) string { switch phase { case "service-apply": return "service apply" + case "schedule-run": + return "schedule run" case "": return "deploy" default: @@ -190,6 +192,8 @@ func auditOutcome(action string) string { return "applied" case "exec": return "succeeded" + case "schedule run": + return "started" default: return "deployed" } diff --git a/internal/engine/deploy_test.go b/internal/engine/deploy_test.go index bf52fb99..7a23f283 100644 --- a/internal/engine/deploy_test.go +++ b/internal/engine/deploy_test.go @@ -59,6 +59,10 @@ func happyFake() *transport.Fake { if strings.Contains(cmd, "Config.Healthcheck") { return transport.Result{Stdout: `{"Test":` + guardedHealthcheck + `,"Interval":5000000000,"Retries":3}` + "\n"}, true } + // The host scheduled jobs need: systemd 252 or newer. + if strings.Contains(cmd, "systemctl --version") { + return transport.Result{Stdout: "systemd 255 (255.4-1ubuntu8)\n"}, true + } // server roll state, derived from history so the loop converges: NEW1 // appears after a scale, OLD1 disappears once removed, names track renames. scaled, oldGone, drained := false, false, false diff --git a/internal/engine/engine.go b/internal/engine/engine.go index d5e75f4a..b667aaae 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -90,10 +90,14 @@ type Engine struct { // wal-g invocation needs to know and which cannot change mid-operation. flockProbed bool flockPresent bool - Compose *ctypes.Project - T transport.Transport - Opts Options - ui *ui.UI + // triggerUnitProbed/triggerUnitPresent cache whether the host's systemd + // tells a timer activation from a manual one (TRIGGER_UNIT, systemd 252). + triggerUnitProbed bool + triggerUnitPresent bool + Compose *ctypes.Project + T transport.Transport + Opts Options + ui *ui.UI // fenceVal is " " once WriteFence has stamped the host; // mutate() guards every mutating command with it. diff --git a/internal/engine/preflight.go b/internal/engine/preflight.go index a78a790d..6e6b6894 100644 --- a/internal/engine/preflight.go +++ b/internal/engine/preflight.go @@ -37,6 +37,15 @@ func (e *Engine) preflight(ctx context.Context, requireDiscovery bool) error { } return app.HostPrerequisiteRefusal("%s: %s", e.T.Host(), unmet.Message) } + // Scheduled jobs have host requirements of their own. Asked here so a + // deploy refuses before staging a release, not after activating it. + jobs, err := e.Spec.ScheduledJobs() + if err != nil { + return err + } + if err := e.requireScheduleHost(ctx, jobs); err != nil { + return err + } base := release.PathsFor(e.names()).Base if res, err := e.T.Run(ctx, "mkdir -p "+q(base)+" && test -w "+q(base)); err != nil || res.ExitCode != 0 { return fmt.Errorf("%s not writable by deploy user", base) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 2be48e80..3421b97f 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -4,8 +4,11 @@ import ( "context" "errors" "fmt" + "math" "regexp" + "strconv" "strings" + "time" "github.com/labstack/onebox/internal/app" "github.com/labstack/onebox/internal/notify" @@ -79,8 +82,8 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { } wanted := map[string]bool{} - if len(jobs) > 0 && !e.hasFlock(ctx) { - return errors.New("scheduled jobs require flock on the target so they cannot overlap deployments; install util-linux and deploy again") + if err := e.requireScheduleHost(ctx, jobs); err != nil { + return err } for _, job := range jobs { unit := n.ScheduledJobUnit(job.Name) @@ -106,8 +109,8 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { if e.Spec.Runtime != nil { runtimeEnvFiles = e.Spec.Runtime.EnvFiles } - runner := scheduleRunnerScript(e.Spec.Name, job, n, e.lockPath(), runtimeEnvFiles) - notifier, err := e.scheduleFailureNotifier(job.Name) + runner := scheduleRunnerScript(e.Spec.Name, job, n, e.lockPath(), runtimeEnvFiles, e.lockTTL(), e.hasTriggerUnit(ctx)) + notifier, err := e.scheduleNotifier(job) if err != nil { return fmt.Errorf("job %s: cannot render its failure notifier: %w", job.Name, err) } @@ -169,85 +172,291 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { // job explicitly opts into the narrower pinned-release contract. Pinned mode // meets the deploy acquirer briefly under schedule.lock, leases the resolved // release before releasing that rendezvous, then retains only its own job lock. -func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile) string { +func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile, lockTTL time.Duration, triggerUnit bool) string { if job.DeployLock == "pinned" { - return pinnedScheduleRunnerScript(application, job.Name, names, applicationLock, runtimeEnvFiles) + return pinnedScheduleRunnerScript(application, job, names, applicationLock, runtimeEnvFiles, lockTTL, triggerUnit) } container := names.Container(job.Name, 1) projectDir := q(names.CurrentLink()) compose := "/usr/bin/docker compose -p " + q(application) + " --project-directory " + projectDir + " -f " + projectDir + "/" + q("compose.yaml") + scheduleRuntimeEnvArgs(projectDir, runtimeEnvFiles) + - " run --rm --no-deps --name " + q(container) + " " + q(job.Name) - return strings.Join([]string{ + " run --rm --no-deps \"$@\" --name " + q(container) + " " + q(job.Name) + lines := []string{ "#!/bin/sh", "# Written by Onebox. Edits are overwritten on the next deploy.", "set -eu", "install -d -m 700 " + q(names.AppDir()+"/schedule"), - "exec 9>" + q(names.ScheduledJobRunLock(job.Name)), - "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 9", - "exec 8>" + q(names.ScheduleRunLock()), - "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 8", - "if [ -e " + q(applicationLock) + " ]; then echo 'onebox: an application operation holds the deploy lock' >&2; exit 75; fi", + } + lines = append(lines, scheduleInputsLines(names.ScheduledJobRunInputs(job.Name))...) + lines = append(lines, scheduleLockLines(names, job.Name, applicationLock, lockTTL)...) + lines = append(lines, + // Best effort: the record names the release that ran, and an exclusive + // job runs whatever `current` points at when it starts. + "release_dir=$(readlink -f "+q(names.CurrentLink())+" 2>/dev/null || true)", + "release=${release_dir##*/}", scheduleContainerCleanup(container), - "cleanup() { " + scheduleContainerCleanup(container) + "; }", + "cleanup() { "+scheduleContainerCleanup(container)+"; rm -f \"$tmp\"; }", "trap cleanup 0", "trap 'exit 129' 1", "trap 'exit 130' 2", "trap 'exit 143' 15", - compose, - "", - }, "\n") + ) + lines = append(lines, scheduleRunPreamble(triggerUnit)...) + lines = append(lines, scheduleAttemptLoop(job, compose, container)...) + lines = append(lines, "") + return strings.Join(lines, "\n") } -func pinnedScheduleRunnerScript(application, job string, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile) string { +func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names app.Names, applicationLock string, runtimeEnvFiles []app.EnvFile, lockTTL time.Duration, triggerUnit bool) string { scheduleDir := names.AppDir() + "/schedule" - state := names.ScheduledJobRunState(job) - container := names.Container(job, 1) + container := names.Container(job.Name, 1) projectDir := `"$release_dir"` compose := "/usr/bin/docker compose -p " + q(application) + " --project-directory " + projectDir + " -f " + projectDir + "/" + q("compose.yaml") + scheduleRuntimeEnvArgs(projectDir, runtimeEnvFiles) + - " run --rm --no-deps --name " + q(container) + " " + q(job) + " run --rm --no-deps \"$@\" --name " + q(container) + " " + q(job.Name) lines := []string{ "#!/bin/sh", "# Written by Onebox. Edits are overwritten on the next deploy.", "set -eu", "install -d -m 700 " + q(scheduleDir), - "exec 9>" + q(names.ScheduledJobRunLock(job)), - "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 9", - "exec 8>" + q(names.ScheduleRunLock()), - "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 8", - "if [ -e " + q(applicationLock) + " ]; then echo 'onebox: an application operation holds the deploy lock' >&2; exit 75; fi", - "release_dir=$(readlink -f " + q(names.CurrentLink()) + ") || { echo 'onebox: current release cannot be resolved' >&2; exit 75; }", - "if [ \"${release_dir%/*}\" != " + q(names.ReleasesDir()) + " ]; then echo 'onebox: current release resolves outside the release store' >&2; exit 75; fi", + } + lines = append(lines, scheduleInputsLines(names.ScheduledJobRunInputs(job.Name))...) + lines = append(lines, scheduleLockLines(names, job.Name, applicationLock, lockTTL)...) + lines = append(lines, + // These are misconfigurations, not timing: the run fails, loudly. + "release_dir=$(readlink -f "+q(names.CurrentLink())+") || { echo 'onebox: current release cannot be resolved' >&2; exit 1; }", + "if [ \"${release_dir%/*}\" != "+q(names.ReleasesDir())+" ]; then echo 'onebox: current release resolves outside the release store' >&2; exit 1; fi", "release=${release_dir##*/}", - "if ! printf '%s\\n' \"$release\" | grep -Eq '^[0-9]{8}-[0-9]{6}-[0-9A-Za-z_-]+$'; then echo 'onebox: current release identity is invalid' >&2; exit 75; fi", - "if [ ! -f \"$release_dir/compose.yaml\" ]; then echo 'onebox: pinned release has no compose.yaml' >&2; exit 75; fi", + "if ! printf '%s\\n' \"$release\" | grep -Eq '^[0-9]{8}-[0-9]{6}-[0-9A-Za-z_-]+$'; then echo 'onebox: current release identity is invalid' >&2; exit 1; fi", + "if [ ! -f \"$release_dir/compose.yaml\" ]; then echo 'onebox: pinned release has no compose.yaml' >&2; exit 1; fi", "exec 7>>\"$release_dir/.ob-schedule.lease\"", "chmod 600 \"$release_dir/.ob-schedule.lease\"", "/usr/bin/flock --shared 7", scheduleContainerCleanup(container), - "state=" + q(state), - "tmp=\"$state.$$\"", - "cleanup() { " + scheduleContainerCleanup(container) + "; rm -f \"$state\" \"$tmp\"; }", + "cleanup() { "+scheduleContainerCleanup(container)+"; rm -f \"$tmp\"; }", "trap cleanup 0", "trap 'exit 129' 1", "trap 'exit 130' 2", "trap 'exit 143' 15", - "started_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", - "umask 077", - "printf 'release=%s\\nstarted_at=%s\\n' \"$release\" \"$started_at\" >\"$tmp\"", - "mv -f \"$tmp\" \"$state\"", - "/usr/bin/flock --unlock 8", - compose, - "", - } + ) + lines = append(lines, scheduleRunPreamble(triggerUnit)...) + // The lease is held; the schedule mutex goes back before the first + // attempt so a compatible deploy is not blocked through the backoff. + lines = append(lines, "/usr/bin/flock --unlock 8") + lines = append(lines, scheduleAttemptLoop(job, compose, container)...) + lines = append(lines, "") return strings.Join(lines, "\n") } +// scheduleLockLines take the run's locks, and turn a conflict into a recorded +// skip rather than a failed unit. A skip is a fact about timing: another run +// of this job, or an application operation, is in progress. The runner writes +// the reason into the state file and exits 0, so systemd sees a clean unit and +// the notifier records `skipped` with that reason. The container's own exit +// status is never mistaken for a skip, because a skip happens before any +// container starts. +// +// The application lock is honoured for as long as AcquireLock would honour it: +// a lock older than the TTL belongs to a runner that died, and AcquireLock +// takes it over, so the timer must not defer to it forever either. The age +// comes from the same shell AcquireLock reads it with, in whole seconds, and +// that shell fails closed: an unreadable lock reads as fresh. +func scheduleLockLines(names app.Names, job, applicationLock string, lockTTL time.Duration) []string { + ttlSeconds := int(math.Ceil(lockTTL.Seconds())) + if ttlSeconds < 1 { + ttlSeconds = 1 + } + return []string{ + "state=" + q(names.ScheduledJobRunState(job)), + "tmp=\"$state.$$\"", + // The operation and inputs of a manual request are kept on the skip + // record too, so `ob schedule run --wait` can find its own outcome. + // Writing the state requires holding the job lock: it is the run in + // flight that owns that file, and overwriting it would replace a real + // run's outcome with this one's skip. + "skip() { umask 077; printf 'skipped=%s\\noperation=%s\\ninputs=%s\\n' \"$1\" \"$operation\" \"$inputs_json\" >\"$tmp\"; mv -f \"$tmp\" \"$state\"; echo \"onebox: skipped: $1\" >&2; exit 0; }", + // No lock, so no claim on the state file: the run already in flight + // owns it and will record itself. This activation leaves its own note + // instead, keyed to its own invocation, and the notifier reads that + // rather than the state a different run is still writing. Without the + // note the notifier would see a clean exit and record this activation + // as a success that never ran. + // Named for the activation, the same way the notifier looks for it. + // The two have to agree exactly: a marker the notifier cannot find + // sends it back to the state file, which is the run in flight's. + "skip_marker=\"$state.skip.${INVOCATION_ID:-}\"", + "stand_aside() { umask 077; printf 'skipped=%s\\noperation=%s\\ninputs=%s\\n' \"$1\" \"$operation\" \"$inputs_json\" >\"$skip_marker\"; echo \"onebox: skipped: $1\" >&2; exit 0; }", + "exec 9>" + q(names.ScheduledJobRunLock(job)), + "/usr/bin/flock --exclusive --nonblock 9 || stand_aside 'another run of this job is still in progress'", + // Only the activation that wrote a note removes it, so one lost + // between the runner exiting and ExecStopPost — a power cut, a killed + // systemd — would sit here forever. Swept a day later, under the job + // lock, which is long past any live note's few milliseconds. + "find " + q(names.AppDir()+"/schedule") + " -maxdepth 1 -name " + q(job+".state.skip.*") + " -mtime +1 -delete 2>/dev/null || true", + "exec 8>" + q(names.ScheduleRunLock()), + "/usr/bin/flock --exclusive --nonblock 8 || skip 'an application operation is taking its lock'", + "if [ -e " + q(applicationLock) + " ] && [ \"$(" + lockAgeCmd(applicationLock) + ")\" -le " + strconv.Itoa(ttlSeconds) + " ]; then skip 'an application operation holds the deploy lock'; fi", + } +} + +// requireScheduleHost is what a host needs before any scheduled job can be +// installed on it. Preflight asks it so a deploy refuses before staging, and +// SyncSchedules asks again so `ob schedule apply` cannot bypass it. +// +// systemd 252 introduced TRIGGER_UNIT, which is how the runner tells a timer +// firing from an operator's start. The floor applies only to a job that +// declares inputs, and to `ob schedule run`; see below for why, and why a host +// that has been running scheduled jobs for years is not refused one. +func (e *Engine) requireScheduleHost(ctx context.Context, jobs []app.ScheduledJob) error { + if len(jobs) == 0 { + return nil + } + if !e.hasFlock(ctx) { + return errors.New("scheduled jobs require flock on the target so they cannot overlap deployments; install util-linux and deploy again") + } + // Declared inputs are the one feature that cannot work without + // TRIGGER_UNIT: the runner would have to guess whether an activation is + // the operator's, and guessing wrong hands a timer firing the inputs a + // person meant for their own run. Everything else works on an older + // systemd, so a host that has run scheduled jobs for years keeps running + // them — it only records `unknown` where it cannot know the trigger. + if !needsTriggerUnit(jobs) || e.hasTriggerUnit(ctx) { + return nil + } + return fmt.Errorf("a job declares inputs, which need systemd 252 or newer on the host: without $TRIGGER_UNIT the runner cannot tell a timer firing from an operator's run") +} + +// hasTriggerUnit reports whether the host's systemd sets TRIGGER_UNIT on a +// timer activation, which systemd 252 introduced. +func (e *Engine) hasTriggerUnit(ctx context.Context) bool { + if e.triggerUnitProbed { + return e.triggerUnitPresent + } + res, err := e.T.Run(ctx, "systemctl --version 2>/dev/null | head -1") + if err != nil { + // A transport failure says nothing about the host's systemd. Caching + // it would turn one flaky round trip into "this host is too old" for + // the rest of the operation, and preflight would refuse the deploy. + return false + } + e.triggerUnitProbed = true + version, ok := systemdVersion(res.Stdout) + e.triggerUnitPresent = ok && version >= 252 + return e.triggerUnitPresent +} + +func needsTriggerUnit(jobs []app.ScheduledJob) bool { + for _, job := range jobs { + if len(job.Inputs) > 0 { + return true + } + } + return false +} + func scheduleContainerCleanup(container string) string { return "/usr/bin/docker rm -f " + q(container) + " >/dev/null 2>&1 || true" } +// scheduleStateFunction renders the shell function both runners use to record +// the run in progress. The notifier reads it after the run ends, so the runner +// never removes it: a runner that cleaned up its own state would erase the +// only evidence a timed-out run leaves behind. +func scheduleStateFunction() []string { + return []string{ + "write_state() {", + " umask 077", + " printf 'release=%s\\nstarted_at=%s\\nstarted_epoch=%s\\ntrigger=%s\\noperation=%s\\nattempt=%s\\ninputs=%s\\n' " + + "\"$release\" \"$started_at\" \"$started_epoch\" \"$trigger\" \"$operation\" \"$1\" \"$inputs_json\" >\"$tmp\"", + " mv -f \"$tmp\" \"$state\"", + "}", + } +} + +// scheduleRunPreamble sets the variables write_state records. The trigger is +// systemd's own word for it: a timer activation carries TRIGGER_UNIT (systemd +// 252 and newer), anything else is an operator. +func scheduleRunPreamble(triggerUnit bool) []string { + // TRIGGER_UNIT is set on a timer activation and on nothing else, so its + // absence names an operator — but only on a systemd that sets it at all. + // On an older host the runner says `unknown` rather than inventing a + // trigger it cannot observe. + otherwise := "unknown" + if triggerUnit { + otherwise = "manual" + } + return append([]string{ + "started_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", + "started_epoch=$(date -u '+%s')", + "if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=" + otherwise + "; fi", + }, scheduleStateFunction()...) +} + +// scheduleInputsLines consumes the one-shot inputs file on a manual +// activation. Values reach the container as -e arguments, never as shell +// text, and the file is gone before any lock is taken so a skipped manual run +// cannot hand its inputs to the next timer firing. A timer activation never +// opens the file: TRIGGER_UNIT says which one this is. +func scheduleInputsLines(inputsPath string) []string { + return []string{ + "operation=''", + "inputs_json=''", + "inputs_file=" + q(inputsPath), + "if [ -z \"${TRIGGER_UNIT:-}\" ] && [ -f \"$inputs_file\" ]; then", + " while IFS= read -r line || [ -n \"$line\" ]; do", + " case \"$line\" in", + " ONEBOX_OPERATION=*) operation=${line#ONEBOX_OPERATION=} ;;", + " [A-Z]*=*) set -- \"$@\" -e \"$line\"; key=${line%%=*}; value=${line#*=}; inputs_json=\"${inputs_json:+$inputs_json,}\\\"$key\\\":\\\"$value\\\"\" ;;", + " esac", + " done <\"$inputs_file\"", + " rm -f \"$inputs_file\"", + "fi", + } +} + +// systemdVersion reads the leading number from `systemd 255 (255.4-1ubuntu8)`. +func systemdVersion(firstLine string) (int, bool) { + fields := strings.Fields(firstLine) + if len(fields) < 2 || fields[0] != "systemd" { + return 0, false + } + n, err := strconv.Atoi(fields[1]) + return n, err == nil +} + +// scheduleAttemptLoop runs the container until it exits 0 or the attempts are +// spent. Backoff doubles and is capped; every sleep happens under the locks +// the run already holds, which is why validation keeps the sum under the +// timeout. A single-attempt job gets no loop, so its runner reads as before. +func scheduleAttemptLoop(job app.ScheduledJob, compose, container string) []string { + if job.RetryAttempts <= 1 { + return []string{"write_state 1", compose} + } + return []string{ + fmt.Sprintf("max_attempts=%d", job.RetryAttempts), + // Whole seconds, the same rounding validation used to bound the sum. + fmt.Sprintf("backoff=%d", app.RetryBackoffSeconds(job.RetryBackoff)), + fmt.Sprintf("max_backoff=%d", app.RetryBackoffSeconds(job.RetryMaxBackoff)), + "attempt=1", + "while :; do", + " write_state \"$attempt\"", + // The container name is fixed, so a corpse from the previous attempt + // would fail every attempt after it with "name already in use" and + // turn one transient failure into all of them. + " " + scheduleContainerCleanup(container), + " status=0", + " " + compose + " || status=$?", + " [ \"$status\" -eq 0 ] && exit 0", + " if [ \"$attempt\" -ge \"$max_attempts\" ]; then exit \"$status\"; fi", + " echo \"onebox: attempt $attempt of $max_attempts exited $status; retrying in ${backoff}s\" >&2", + " sleep \"$backoff\"", + " backoff=$((backoff * 2))", + " [ \"$backoff\" -gt \"$max_backoff\" ] && backoff=$max_backoff", + " attempt=$((attempt + 1))", + "done", + } +} + func scheduleRuntimeEnvArgs(projectDir string, entries []app.EnvFile) string { args := "" for _, entry := range entries { @@ -270,9 +479,9 @@ func scheduleServiceUnit(application string, job app.ScheduledJob, runnerPath, n "[Service]", "Type=oneshot", "ExecStart=/bin/sh " + runnerPath, - // ExecStopPost runs after success, start failures, and timeouts. It always - // attempts fenced container cleanup, then uses SERVICE_RESULT to decide - // whether failure notifications are needed. + // ExecStopPost runs after success, failures, and timeouts. It always + // attempts fenced container cleanup, writes the run record from the + // runner's state and SERVICE_RESULT, then notifies per the job's policy. "ExecStopPost=/bin/sh " + notifyPath, "TimeoutStartSec=" + job.Timeout, "", @@ -281,10 +490,96 @@ func scheduleServiceUnit(application string, job app.ScheduledJob, runnerPath, n const scheduleNotificationTimestamp = "__ONEBOX_SCHEDULE_TIMESTAMP__" -// scheduleFailureNotifier extends the existing notification contract to work -// fired directly by systemd. The generated file is mode 0600, keeping webhook +// scheduleRunIdentifier is the syslog identifier of the one line the notifier +// writes per run. The journal is the store, so there is no file to trim and +// nothing that can disagree with the unit's own log. +// +// The line carries its own ONEBOX_UNIT and ONEBOX_JOB fields and the history +// query matches on them, not on journald's cgroup attribution. A process that +// writes one line and exits is often gone before journald reads /proc for it, +// and such an entry has no _SYSTEMD_UNIT at all; `journalctl -u` would never +// find it. Explicit fields survive that race, and `logger --journald` is +// util-linux, which flock already requires. +const scheduleRunIdentifier = "ob-run" + +// scheduleRunRecordLines finalises the run the runner started. This lives in +// ExecStopPost because only systemd knows how the run ended: a timed-out +// runner is killed mid-sleep and cannot write its own outcome. Every value +// interpolated into the JSON is either numeric, a timestamp the runner +// formatted, a release id, or an input value the loader restricted to a +// charset that needs no escaping. +func scheduleRunRecordLines(application, unit, job, state string) []string { + return []string{ + "state=" + q(state), + "release=''; started_at=''; started_epoch=''; trigger=''; operation=''; attempt=0; inputs=''; skipped=''", + // A run that stood aside left a note under its own invocation. It + // never held the job lock, so the state file belongs to whichever run + // is still going: read the note and leave that file alone. + "skip_marker=\"$state.skip.${INVOCATION_ID:-}\"", + "if [ -f \"$skip_marker\" ]; then", + " while IFS= read -r line || [ -n \"$line\" ]; do", + " case \"$line\" in", + " skipped=*) skipped=${line#skipped=} ;;", + " operation=*) operation=${line#operation=} ;;", + " inputs=*) inputs=${line#inputs=} ;;", + " esac", + " done <\"$skip_marker\"", + " rm -f \"$skip_marker\"", + "elif [ -f \"$state\" ]; then", + " while IFS= read -r line || [ -n \"$line\" ]; do", + " case \"$line\" in", + " release=*) release=${line#release=} ;;", + " started_at=*) started_at=${line#started_at=} ;;", + " started_epoch=*) started_epoch=${line#started_epoch=} ;;", + " trigger=*) trigger=${line#trigger=} ;;", + " operation=*) operation=${line#operation=} ;;", + " attempt=*) attempt=${line#attempt=} ;;", + " inputs=*) inputs=${line#inputs=} ;;", + " skipped=*) skipped=${line#skipped=} ;;", + " esac", + " done <\"$state\"", + " rm -f \"$state\"", + "fi", + "if [ -z \"$trigger\" ]; then if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=unknown; fi; fi", + "result=${SERVICE_RESULT:-success}", + "status=${EXIT_STATUS:-0}", + // EXIT_STATUS is a signal name when the main process was killed. + "case \"$status\" in ''|*[!0-9]*) status=null ;; esac", + "case \"$attempt\" in ''|*[!0-9]*) attempt=0 ;; esac", + // A skip is the runner's own word, written before any container ran; + // a container that exits non-zero, 75 included, is a failure. + "if [ \"$result\" = timeout ]; then outcome=timeout", + "elif [ -n \"$skipped\" ]; then outcome=skipped", + "elif [ \"$result\" = success ] && [ \"$status\" = 0 ]; then outcome=success", + "else outcome=failure; fi", + "finished_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", + "now=$(date -u '+%s')", + "duration=0", + "case \"$started_epoch\" in ''|*[!0-9]*) ;; *) duration=$((now - started_epoch)) ;; esac", + "[ -z \"$started_at\" ] && started_at=$finished_at", + "record=$(printf '{\"run\":\"%s\",\"job\":\"%s\",\"trigger\":\"%s\",\"operation\":\"%s\",\"release\":\"%s\",\"started_at\":\"%s\",\"finished_at\":\"%s\",\"duration_s\":%s,\"attempts\":%s,\"exit_status\":%s,\"outcome\":\"%s\",\"reason\":\"%s\",\"inputs\":{%s}}' " + + "\"${INVOCATION_ID:-}\" " + q(job) + " \"$trigger\" \"$operation\" \"$release\" \"$started_at\" \"$finished_at\" \"$duration\" \"$attempt\" \"$status\" \"$outcome\" \"$skipped\" \"$inputs\")", + "printf 'MESSAGE=%s\\nPRIORITY=6\\nSYSLOG_IDENTIFIER=" + scheduleRunIdentifier + "\\nONEBOX_APP=%s\\nONEBOX_UNIT=%s\\nONEBOX_JOB=%s\\n' " + + "\"$record\" " + q(application) + " " + q(unit) + " " + q(job) + " | logger --journald || true", + } +} + +// scheduleNotificationRun marks where the notifier substitutes the run id at +// send time. It travels as the payload's deploy_id: the correlation key an +// operator hands to `ob schedule logs --run`. Nothing else about the run goes +// into a notification; the notify package redacts diagnostics on purpose, and +// attempts, duration and exit status belong to the run record on the host. +const scheduleNotificationRun = "__ONEBOX_SCHEDULE_RUN__" + +// scheduleNotifier extends the existing notification contract to work fired +// directly by systemd. It finalises the run record first, then sends for the +// outcomes the job selected. The generated file is mode 0600, keeping webhook // tokens out of unit metadata, and every send is bounded and fail-open. -func (e *Engine) scheduleFailureNotifier(job string) (string, error) { +// +// Bodies are prepared here, once per outcome class, because the payload +// contract lives in the notify package and the host has no Onebox to ask at +// 2am. Only the timestamp and the run id are filled in on the host. +func (e *Engine) scheduleNotifier(job app.ScheduledJob) (string, error) { environment := e.Opts.Environment if environment == "" { environment = e.Spec.Env @@ -293,45 +588,121 @@ func (e *Engine) scheduleFailureNotifier(job string) (string, error) { "#!/bin/sh", "# Written by Onebox. Edits are overwritten on the next deploy.", "set -u", - "exec 9>" + q(e.names().ScheduledJobRunLock(job)), + "exec 9>" + q(e.names().ScheduledJobRunLock(job.Name)), "if /usr/bin/flock --exclusive --nonblock 9; then", - " " + scheduleContainerCleanup(e.names().Container(job, 1)), + " " + scheduleContainerCleanup(e.names().Container(job.Name, 1)), "fi", - `[ "${SERVICE_RESULT:-success}" = success ] && exit 0`, } + lines = append(lines, scheduleRunRecordLines(e.Spec.Name, e.names().ScheduledJobUnit(job.Name), job.Name, e.names().ScheduledJobRunState(job.Name))...) + lines = append(lines, `case " `+strings.Join(job.Notify, " ")+` " in *" $outcome "*) ;; *) exit 0 ;; esac`) + // Three classes, because a skip is neither: the job did not fail, and it + // did not do its work either. + var wants = map[string]bool{} + for _, outcome := range job.Notify { + switch outcome { + case "success": + wants["ok"] = true + case "skipped": + wants["skipped"] = true + default: + wants["fail"] = true + } + } + sends := map[string][]string{} + for _, class := range []string{"ok", "skipped", "fail"} { + if !wants[class] { + continue + } + rendered, err := e.scheduleNotificationSends(job.Name, environment, class) + if err != nil { + return "", err + } + sends[class] = rendered + } + if len(sends["ok"])+len(sends["skipped"])+len(sends["fail"]) > 0 { + lines = append(lines, `ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ')`) + lines = append(lines, `if [ "$outcome" = success ]; then`) + lines = append(lines, orNoop(sends["ok"])...) + lines = append(lines, `elif [ "$outcome" = skipped ]; then`) + lines = append(lines, orNoop(sends["skipped"])...) + lines = append(lines, "else") + lines = append(lines, orNoop(sends["fail"])...) + lines = append(lines, "fi", "wait || true") + } + lines = append(lines, "exit 0", "") + return strings.Join(lines, "\n"), nil +} + +// orNoop keeps a shell branch syntactically present when it has nothing to do. +func orNoop(lines []string) []string { + if len(lines) == 0 { + return []string{" :"} + } + return lines +} + +// scheduleNotificationSends renders one backgrounded curl per notification +// that selects the given class: ok, skipped, or fail. A skip routes with the +// failures, because that is the channel an operator watches, and says what it +// is rather than claiming the job failed. A failed send is logged and never +// replaces the job's own result. +func (e *Engine) scheduleNotificationSends(job, environment, class string) ([]string, error) { var sends []string for _, name := range sortedNames(e.Spec.Notifications) { cfg := e.Spec.Notifications[name] - prepared, err := notify.Prepare(cfg, notify.Payload{ + status := "fail" + if class == "ok" { + status = "ok" + } + payload := notify.Payload{ App: e.Spec.Name, Env: environment, Host: e.T.Destination(), - Verb: "scheduled job " + job, Status: "fail", - Error: "scheduled job failed; inspect trusted host diagnostics", - TS: scheduleNotificationTimestamp, - }) + Verb: "scheduled job " + job, Status: status, + DeployID: scheduleNotificationRun, TS: scheduleNotificationTimestamp, + Skipped: class == "skipped", + } + if status != "ok" { + payload.Error = "scheduled job failed; inspect trusted host diagnostics" + } + prepared, err := notify.Prepare(cfg, payload) if err != nil { - return "", err + return nil, err } if prepared == nil { continue } - body := q(string(prepared.Body)) - if before, after, ok := strings.Cut(string(prepared.Body), scheduleNotificationTimestamp); ok { - body = q(before) + `"$ts"` + q(after) - } curl := "curl --fail --silent --show-error --max-time 5 --request POST" + " --header " + q("Content-Type: "+prepared.ContentType) + " --header " + q("X-Title: "+prepared.Title) + ` --data-binary "$body" ` + q(cfg.Webhook) - sends = append(sends, "(body="+body+"; if ! "+curl+"; then echo "+ + sends = append(sends, " (body="+shellBody(string(prepared.Body))+"; if ! "+curl+"; then echo "+ q("onebox: notification "+name+" failed")+" >&2; fi) &") } - if len(sends) > 0 { - lines = append(lines, `ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ')`) - lines = append(lines, sends...) - lines = append(lines, "wait || true") + return sends, nil +} + +// shellBody quotes a prepared body for the shell, leaving the two runtime +// placeholders as expansions of variables the notifier sets before sending. +func shellBody(body string) string { + var out strings.Builder + for body != "" { + next, placeholder, expansion := -1, "", "" + if i := strings.Index(body, scheduleNotificationTimestamp); i >= 0 { + next, placeholder, expansion = i, scheduleNotificationTimestamp, `"$ts"` + } + if i := strings.Index(body, scheduleNotificationRun); i >= 0 && (next < 0 || i < next) { + next, placeholder, expansion = i, scheduleNotificationRun, `"${INVOCATION_ID:-}"` + } + if next < 0 { + out.WriteString(q(body)) + break + } + if next > 0 { + out.WriteString(q(body[:next])) + } + out.WriteString(expansion) + body = body[next+len(placeholder):] } - lines = append(lines, "exit 0", "") - return strings.Join(lines, "\n"), nil + return out.String() } // calendarExpr is the one string both the host's validator and the installed diff --git a/internal/engine/schedule_history.go b/internal/engine/schedule_history.go new file mode 100644 index 00000000..2ae4b535 --- /dev/null +++ b/internal/engine/schedule_history.go @@ -0,0 +1,201 @@ +package engine + +import ( + "context" + "encoding/json" + "fmt" + "io" + "regexp" + "strconv" + "strings" + + "github.com/labstack/onebox/internal/app" +) + +// ScheduleRunRecord is the line the notifier writes to the journal when a +// scheduled run ends. The journal is the store: there is no file to trim and +// nothing that can disagree with the unit's own log. +type ScheduleRunRecord struct { + Run string `json:"run"` + Job string `json:"job"` + Trigger string `json:"trigger"` + Operation string `json:"operation,omitempty"` + Release string `json:"release,omitempty"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + DurationSeconds int `json:"duration_s"` + Attempts int `json:"attempts"` + ExitStatus *int `json:"exit_status"` + Outcome string `json:"outcome"` + // Reason is set on a skipped run: what the runner met instead of running. + Reason string `json:"reason,omitempty"` + Inputs map[string]string `json:"inputs,omitempty"` +} + +// ScheduleListing is one declared job beside its timer as the host reports it. +type ScheduleListing struct { + Name string `json:"name"` + Unit string `json:"unit"` + Cron string `json:"cron"` + Timezone string `json:"timezone"` + DeployLock string `json:"deploy_lock"` + Timeout string `json:"timeout"` + TimerState string `json:"timer_state"` + NextRun string `json:"next_run,omitempty"` + LastTrigger string `json:"last_trigger,omitempty"` +} + +// A run id is systemd's invocation id. It reaches a shell as a journalctl +// match, so it is checked against the only shape systemd produces. +var scheduleRunID = regexp.MustCompile(`^[0-9a-f]{32}$`) + +// scheduleHistoryCommand matches the record's own fields rather than the +// unit journald attributed it to; see scheduleRunIdentifier for why. A read +// that fails is left to fail: an empty history and an unreadable journal are +// different answers, and only one of them means the job never ran. +func scheduleHistoryCommand(unit string, n int) string { + if n <= 0 { + n = 20 + } + return "journalctl SYSLOG_IDENTIFIER=" + scheduleRunIdentifier + " ONEBOX_UNIT=" + q(unit) + + " -o cat -r -n " + strconv.Itoa(n) + " --no-pager" +} + +// scheduleOutcomes is the closed vocabulary the notifier writes. A record +// claiming anything else did not come from a runner this binary generated. +var scheduleOutcomes = map[string]bool{"success": true, "failure": true, "timeout": true, "skipped": true} + +// parseScheduleRunRecords keeps the lines that are records of this job's runs +// and drops the rest. Decoding is not enough on its own: callers read the +// first entry as the newest run and let it stand for the job's health, so a +// stray JSON line that happened to be logged under the same identifier could +// clear a failure. A record has to name this job, carry a systemd invocation +// id, and report an outcome from the closed set before it counts. +func parseScheduleRunRecords(stdout, job string) []ScheduleRunRecord { + var out []ScheduleRunRecord + for _, line := range strings.Split(stdout, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "{") { + continue + } + var record ScheduleRunRecord + if err := json.Unmarshal([]byte(line), &record); err != nil { + continue + } + if record.Job != job || !scheduleRunID.MatchString(record.Run) || !scheduleOutcomes[record.Outcome] { + continue + } + out = append(out, record) + } + return out +} + +func (e *Engine) scheduledJob(name string) (app.ScheduledJob, error) { + jobs, err := e.Spec.ScheduledJobs() + if err != nil { + return app.ScheduledJob{}, err + } + for _, job := range jobs { + if job.Name == name { + return job, nil + } + } + return app.ScheduledJob{}, fmt.Errorf("job %q is not a scheduled job", name) +} + +// ScheduleHistory returns the newest n run records of one job, newest first. +func (e *Engine) ScheduleHistory(ctx context.Context, name string, n int) ([]ScheduleRunRecord, error) { + job, err := e.scheduledJob(name) + if err != nil { + return nil, err + } + res, err := e.T.Run(ctx, scheduleHistoryCommand(e.names().ScheduledJobUnit(job.Name), n)) + if err != nil { + return nil, err + } + if res.ExitCode != 0 { + return nil, fmt.Errorf("read run records of %s from the host journal (exit %d): %s", name, res.ExitCode, strings.TrimSpace(res.Stderr)) + } + records := parseScheduleRunRecords(res.Stdout, job.Name) + if records == nil { + records = []ScheduleRunRecord{} + } + return records, nil +} + +// ScheduleList reads every declared job's timer in one round trip. +func (e *Engine) ScheduleList(ctx context.Context) ([]ScheduleListing, error) { + jobs, err := e.Spec.ScheduledJobs() + if err != nil { + return nil, err + } + if len(jobs) == 0 { + return []ScheduleListing{}, nil + } + var commands []string + for _, job := range jobs { + unit := e.names().ScheduledJobUnit(job.Name) + commands = append(commands, + "printf '%s\\n' "+q("@@"+job.Name), + // Left to fail: the exit check below distinguishes a timer with + // nothing to say from a host that would not answer, and `|| true` + // would make that check unreachable. + "systemctl show "+q(unit+".timer")+" --no-pager --property=ActiveState --property=NextElapseUSecRealtime --property=LastTriggerUSec") + } + res, err := e.T.Run(ctx, strings.Join(commands, "\n")) + if err != nil { + return nil, err + } + if res.ExitCode != 0 { + // Blank timer state and "the host would not tell us" are different + // answers, and a table of dashes must not stand for the second. + return nil, fmt.Errorf("read scheduled-job timers (exit %d): %s", res.ExitCode, strings.TrimSpace(res.Stderr)) + } + observed := map[string]map[string]string{} + current := "" + for _, line := range strings.Split(res.Stdout, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "@@") { + current = strings.TrimPrefix(line, "@@") + observed[current] = map[string]string{} + continue + } + if key, value, ok := strings.Cut(line, "="); ok && current != "" { + observed[current][key] = value + } + } + out := make([]ScheduleListing, 0, len(jobs)) + for _, job := range jobs { + values := observed[job.Name] + out = append(out, ScheduleListing{ + Name: job.Name, Unit: e.names().ScheduledJobUnit(job.Name), Cron: job.Cron, Timezone: job.Timezone, + DeployLock: job.DeployLock, Timeout: job.Timeout, TimerState: values["ActiveState"], + NextRun: values["NextElapseUSecRealtime"], LastTrigger: values["LastTriggerUSec"], + }) + } + return out, nil +} + +// ScheduleLogs streams the journal of one run and returns the run id it +// streamed. The run id is systemd's invocation id, so the output is exactly +// that activation and nothing else. With no run given, the newest record's run +// is used, and the caller learns which one that was. +func (e *Engine) ScheduleLogs(ctx context.Context, name, run string, stdout, stderr io.Writer) (string, error) { + if _, err := e.scheduledJob(name); err != nil { + return "", err + } + if run == "" { + records, err := e.ScheduleHistory(ctx, name, 1) + if err != nil { + return "", err + } + if len(records) == 0 { + return "", fmt.Errorf("job %s has no recorded runs", name) + } + run = records[0].Run + } + if !scheduleRunID.MatchString(run) { + return "", fmt.Errorf("run id %q is not a systemd invocation id", run) + } + return run, e.T.RunStream(ctx, "journalctl _SYSTEMD_INVOCATION_ID="+run+" --no-pager -o short-iso", stdout, stderr) +} diff --git a/internal/engine/schedule_history_test.go b/internal/engine/schedule_history_test.go new file mode 100644 index 00000000..462c59a0 --- /dev/null +++ b/internal/engine/schedule_history_test.go @@ -0,0 +1,176 @@ +package engine + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/transport" +) + +const sampleRunRecords = `{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"manual","operation":"20260905-151200-schedule_run-7c1e","release":"20260905-140000-ab12cd","started_at":"2026-09-05T15:12:01Z","finished_at":"2026-09-05T15:12:31Z","duration_s":30,"attempts":2,"exit_status":0,"outcome":"success","inputs":{"SOURCE":"prices"}} +not json +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","operation":"","release":"20260905-140000-ab12cd","started_at":"2026-09-05T15:00:01Z","finished_at":"2026-09-05T15:00:02Z","duration_s":1,"attempts":0,"exit_status":75,"outcome":"skipped","inputs":{}} +` + +func TestParseScheduleRunRecordsSkipsNoiseAndKeepsOrder(t *testing.T) { + records := parseScheduleRunRecords(sampleRunRecords, "nightly") + if len(records) != 2 { + t.Fatalf("records = %#v", records) + } + if records[0].Run != "b2c3d4e5f60718293a4b5c6d7e8f9012" || records[0].Trigger != "manual" || records[0].Attempts != 2 || + records[0].DurationSeconds != 30 || records[0].Inputs["SOURCE"] != "prices" || records[0].ExitStatus == nil || *records[0].ExitStatus != 0 { + t.Fatalf("first record was not decoded: %#v", records[0]) + } + if records[1].Outcome != "skipped" || *records[1].ExitStatus != 75 { + t.Fatalf("second record was not decoded: %#v", records[1]) + } +} + +func scheduledFixture(t *testing.T) (*Engine, *transport.Fake) { + t.Helper() + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{} + return New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}), f +} + +func TestScheduleHistoryReadsTheUnitJournalNewestFirst(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "journalctl") { + return transport.Result{Stdout: sampleRunRecords}, true + } + return transport.Result{}, false + } + records, err := e.ScheduleHistory(context.Background(), "nightly", 20) + if err != nil { + t.Fatal(err) + } + if len(records) != 2 || records[0].Outcome != "success" { + t.Fatalf("records = %#v", records) + } + seq := strings.Join(f.Commands, "\n") + for _, want := range []string{"journalctl SYSLOG_IDENTIFIER=ob-run ONEBOX_UNIT='ob-sample-nightly'", "-o cat", "-r", "-n 20", "--no-pager"} { + if !strings.Contains(seq, want) { + t.Fatalf("history read is missing %q:\n%s", want, seq) + } + } + if _, err := e.ScheduleHistory(context.Background(), "web", 20); err == nil { + t.Fatal("history of a non-scheduled workload was not refused") + } +} + +func TestScheduleListReadsTimerState(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: "@@nightly\nActiveState=active\nNextElapseUSecRealtime=Sat 2026-09-06 02:00:00 UTC\nLastTriggerUSec=Fri 2026-09-05 02:00:00 UTC\n"}, true + } + return transport.Result{}, false + } + listing, err := e.ScheduleList(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(listing) != 1 || listing[0].Unit != "ob-sample-nightly" || listing[0].TimerState != "active" || + listing[0].NextRun != "Sat 2026-09-06 02:00:00 UTC" || listing[0].LastTrigger != "Fri 2026-09-05 02:00:00 UTC" || listing[0].Cron != "0 2 * * *" { + t.Fatalf("listing = %#v", listing) + } +} + +func TestScheduleLogsTargetsOneInvocation(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "SYSLOG_IDENTIFIER=ob-run") { + return transport.Result{Stdout: sampleRunRecords}, true + } + return transport.Result{}, false + } + var out bytes.Buffer + run, err := e.ScheduleLogs(context.Background(), "nightly", "", &out, &out) + if err != nil { + t.Fatal(err) + } + if run != "b2c3d4e5f60718293a4b5c6d7e8f9012" { + t.Fatalf("resolved run id = %q; the caller must learn which run was streamed", run) + } + seq := strings.Join(f.Commands, "\n") + if !strings.Contains(seq, "journalctl _SYSTEMD_INVOCATION_ID=b2c3d4e5f60718293a4b5c6d7e8f9012") { + t.Fatalf("logs did not target the newest run:\n%s", seq) + } + if _, err := e.ScheduleLogs(context.Background(), "nightly", "../etc", &out, &out); err == nil { + t.Fatal("an invalid run id reached the shell") + } +} + +func TestScheduleLogsRefusesAJobWithoutRecords(t *testing.T) { + e, f := scheduledFixture(t) + var out bytes.Buffer + _, err := e.ScheduleLogs(context.Background(), "nightly", "", &out, &out) + if err == nil || !strings.Contains(err.Error(), "no recorded runs") { + t.Fatalf("err = %v", err) + } + for _, command := range f.Commands { + if strings.Contains(command, "_SYSTEMD_INVOCATION_ID") || strings.Contains(command, "journalctl -u") { + t.Fatalf("logs guessed a source without a record: %s", command) + } + } +} + +func TestScheduleHistorySurfacesAJournalReadFailure(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "journalctl") { + return transport.Result{ExitCode: 1, Stderr: "Failed to open journal: Permission denied"}, true + } + return transport.Result{}, false + } + _, err := e.ScheduleHistory(context.Background(), "nightly", 20) + if err == nil || !strings.Contains(err.Error(), "Permission denied") { + t.Fatalf("an unreadable journal was reported as an empty history: %v", err) + } +} + +// Callers read the newest record as the job's standing verdict, so a line that +// merely happens to be JSON must not become one. +func TestParseScheduleRunRecordsRejectsLinesThatAreNotThisJobsRuns(t *testing.T) { + good := `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:02Z","duration_s":1,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}}` + for name, line := range map[string]string{ + "another job": `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"other","outcome":"success"}`, + "no run id": `{"run":"","job":"nightly","outcome":"success"}`, + "short run id": `{"run":"a1b2","job":"nightly","outcome":"success"}`, + "unknown outcome": `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","outcome":"cancelled"}`, + "unrelated json": `{"level":"info","msg":"something else"}`, + } { + t.Run(name, func(t *testing.T) { + records := parseScheduleRunRecords(line+"\n"+good, "nightly") + if len(records) != 1 || records[0].Outcome != "success" { + t.Fatalf("a line that is not this job's run was kept: %#v", records) + } + }) + } +} + +// A timer read that fails is not a timer with nothing to say. Dashes in the +// table must not stand for a host that would not answer. +func TestScheduleListSurfacesAFailedTimerRead(t *testing.T) { + e, f := scheduledFixture(t) + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{ExitCode: 1, Stderr: "Failed to connect to bus"}, true + } + return transport.Result{}, false + } + if _, err := e.ScheduleList(context.Background()); err == nil || !strings.Contains(err.Error(), "Failed to connect to bus") { + t.Fatalf("an unreadable timer was reported as blank state: %v", err) + } + if seq := strings.Join(f.Commands, "\n"); strings.Contains(seq, "|| true") { + t.Fatalf("the read swallows its own failure:\n%s", seq) + } +} diff --git a/internal/engine/schedule_run.go b/internal/engine/schedule_run.go new file mode 100644 index 00000000..4414b3b7 --- /dev/null +++ b/internal/engine/schedule_run.go @@ -0,0 +1,263 @@ +package engine + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/journal" +) + +// ScheduleRunResult is what an operator-initiated run leaves on the +// workstation side. The outcome lives on the host: it is the run record, and +// it is returned here only when the caller waited for it. +type ScheduleRunResult struct { + Job string `json:"job"` + Unit string `json:"unit"` + Operation string `json:"operation"` + Inputs map[string]string `json:"inputs,omitempty"` + Started bool `json:"started"` + Record *ScheduleRunRecord `json:"record,omitempty"` +} + +// ScheduleRun starts a scheduled job's unit now with declared, validated +// inputs. It is journaled like every operation, but the application lock is +// released before the unit starts: the runner exits 75 when it sees that +// lock, so holding it across the start would skip the very run requested. +// +// Only a job whose data effect is none may run this way. The timer already +// runs any scheduled job unattended, but an operator choosing the moment and +// the inputs is the case the sealed plan of `ob job run` exists for, and a +// migration or destructive job keeps that path. +func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inputs map[string]string, wait bool) (_ ScheduleRunResult, err error) { + result := ScheduleRunResult{Job: name, Operation: operationID, Inputs: inputs} + if strings.TrimSpace(operationID) == "" { + return result, errors.New("schedule run requires an operation id") + } + if err := e.RequireHostOwner(ctx); err != nil { + return result, err + } + if _, err := e.scheduledJob(name); err != nil { + return result, err + } + workload := e.Spec.Workloads[name] + if workload.DataEffect != app.DataEffectNone { + return result, fmt.Errorf("job %s declares data_effect %q; operator-initiated runs of it go through the sealed plan: ob job plan %s, then ob job run", + name, workload.DataEffect, name) + } + if err := app.ValidateJobInputValues(workload, inputs); err != nil { + return result, err + } + // The inputs file is addressed to this activation and to no other. Only a + // host whose systemd sets TRIGGER_UNIT lets the runner tell them apart; on + // an older one the next timer firing would read the file meant for this + // run, and the run itself would be recorded as a firing. + if !e.hasTriggerUnit(ctx) { + return result, errors.New("this host's systemd does not set $TRIGGER_UNIT, so a timer firing cannot be told from this run; ob schedule run needs systemd 252 or newer. The timer itself keeps working") + } + unit := e.names().ScheduledJobUnit(name) + result.Unit = unit + + // Starting an active unit is a no-op to systemd and would consume the + // inputs file for a run that never happens; say so instead. + active, err := e.T.Run(ctx, "systemctl is-active "+q(unit+".service")+" 2>/dev/null || true") + if err != nil { + return result, err + } + switch state := strings.TrimSpace(active.Stdout); state { + case "active", "activating", "deactivating": + return result, fmt.Errorf("job %s is running (%s); wait for it, or read ob schedule history %s", name, state, name) + } + + epoch, err := e.AcquireLock(ctx, operationID, e.Opts.ForceLock) + if err != nil { + return result, err + } + locked := true + defer func() { + if locked { + e.ReleaseLock(ctx) + } + }() + if err := e.WriteFence(ctx, operationID, epoch); err != nil { + return result, err + } + + // noclobber: a second manual run before the first is consumed would + // otherwise rewrite the file under it and misattribute the inputs. The + // existence check in front gives that case its own exit status, so a + // host that simply refuses the write is reported as that and not as a + // pending run nobody can find. + path := e.names().ScheduledJobRunInputs(name) + create := "if [ -e " + q(path) + " ]; then exit 73; fi; " + + "umask 077 && install -d -m 700 " + q(e.names().AppDir()+"/schedule") + " && set -C && cat > " + q(path) + res, err := e.mutateInput(ctx, create, scheduleInputsFile(operationID, inputs)) + if err != nil { + return result, err + } + switch { + case res.ExitCode == 73: + return result, fmt.Errorf("a manual run of %s is already pending (%s exists); wait for it, or remove the file on the host", name, path) + case res.ExitCode != 0: + return result, fmt.Errorf("cannot write the inputs file %s on the host: %s", path, strings.TrimSpace(res.Stderr)) + } + // From here on the file is ours to clean up: a request that fails before + // the unit starts must not leave it behind to refuse the next one. + pending := true + defer func() { + if pending { + e.discardInputs(ctx, path) + } + }() + + writer := &journal.Writer{ + T: e.T, Names: e.names(), DeployID: operationID, Epoch: epoch, Operator: journal.DefaultOperator(), + GitSHA: e.Opts.GitSHA, ConfigHash: e.Opts.ConfigHash, Runner: &e.Opts.Runner, + } + detail := "inputs: defaults" + if len(inputs) > 0 { + detail = "inputs: " + scheduleInputsDetail(inputs) + } + record := journal.Record{Phase: "schedule-run", Event: "start", Status: "ok", Target: name, TargetKind: "job", Detail: detail} + if err := writer.Append(ctx, record); err != nil { + return result, fmt.Errorf("journal schedule run start: %w", err) + } + // The finish records how this request ended, not how the run did: the run + // has its own record on the host, joined to this entry by the operation + // id. But a request that never started the unit, or that waited and saw + // the job fail, is not a success, and `ob audit` has to be able to say so. + // + // Written after the lock is released, which is safe because a journal is + // per operation id: no other operation appends to this file. + defer func() { + finish := record + finish.Event, finish.Status = "finish", "ok" + finish.Detail = "unit started; outcome in ob schedule history " + name + if err != nil { + finish.Status, finish.Detail = "fail", err.Error() + } + if appendErr := writer.Append(ctx, finish); appendErr != nil { + err = errors.Join(err, fmt.Errorf("journal schedule run finish: %w", appendErr)) + } + }() + e.ReleaseLock(ctx) + locked = false + + start := "systemctl start --no-block " + q(unit+".service") + if wait { + start = "systemctl start " + q(unit+".service") + } + res, err = e.mutate(ctx, start) + if err != nil { + return result, err + } + if !wait { + if res.ExitCode != 0 { + return result, fmt.Errorf("systemctl start %s: %s", unit, strings.TrimSpace(res.Stderr)) + } + // The unit is queued, so the runner owns the file now. + pending = false + result.Started = true + e.logf("schedule: %s started as %s; ob schedule history %s shows the outcome", name, operationID, name) + return result, nil + } + // A blocking start that exits non-zero may mean the job failed, which is + // an outcome, or that the unit never activated, which is not. Only the + // record settles it, and only a record carrying this operation says the + // runner read the inputs file: a start that merged into a timer firing + // already in progress leaves that file untouched, for the next manual run + // that would otherwise be refused as pending. + last, err := e.awaitScheduleRecord(ctx, name, operationID) + if err != nil { + if res.ExitCode != 0 { + return result, fmt.Errorf("%w; systemctl start exited %d: %s", err, res.ExitCode, strings.TrimSpace(res.Stderr)) + } + return result, err + } + pending = false + result.Started = true + result.Record = last + exit := "-" + if last.ExitStatus != nil { + exit = fmt.Sprint(*last.ExitStatus) + } + e.logf("schedule: %s run %s %s after %d attempt(s) in %ds (exit %s)", + name, last.Run, last.Outcome, last.Attempts, last.DurationSeconds, exit) + // The operator asked for this run and waited for it, so anything but a + // success is a failure of the request, a skip included: the unit exits + // cleanly, but the work was not done. + if last.Outcome != "success" { + return result, fmt.Errorf("job %s run %s ended %s; see ob schedule logs %s --run %s", name, last.Run, last.Outcome, name, last.Run) + } + return result, nil +} + +// awaitScheduleRecord finds the record of this operation's run. The notifier +// writes it from ExecStopPost and journald ingests it a moment after the +// blocking start returns, so a few short retries stand between the start and +// the read. Matching on the operation id means a record left by an earlier +// run, or by a timer firing that took this slot, is never reported as ours. +// The window is generous because journald ingests the notifier's line +// asynchronously: the blocking start has returned, so ExecStopPost has run, +// but the record may not be queryable yet on a loaded host. Giving up early +// would blame the host for a run that in fact succeeded. +func (e *Engine) awaitScheduleRecord(ctx context.Context, name, operationID string) (*ScheduleRunRecord, error) { + for attempt := range 40 { + if attempt > 0 { + e.Opts.Sleep(250 * time.Millisecond) + } + records, err := e.ScheduleHistory(ctx, name, 5) + if err != nil { + return nil, err + } + for i := range records { + if records[i].Operation == operationID { + return &records[i], nil + } + } + } + return nil, fmt.Errorf("no run record for operation %s appeared within %s: the run may still be settling in the host journal, a timer firing may have taken the slot, or the notifier wrote nothing. ob schedule history %s shows what the host has", + operationID, 10*time.Second, name) +} + +// discardInputs removes a pending inputs file this request wrote and can no +// longer hand to a run. Best effort: the file is root-only state on the host, +// and the error the caller is already returning is the one that matters. +func (e *Engine) discardInputs(ctx context.Context, path string) { + if _, err := e.T.Run(ctx, "rm -f "+q(path)); err != nil { + e.warnf("could not remove the pending inputs file %s: %v", path, err) + } +} + +// scheduleInputsFile is the one-shot file the runner consumes: the operation +// id on its reserved line, then one declared override per line. Values were +// validated against a charset that has no newline or quote, so the format +// needs no escaping. +func scheduleInputsFile(operationID string, inputs map[string]string) string { + lines := []string{app.ReservedInputPrefix + "OPERATION=" + operationID} + for _, name := range sortedInputNames(inputs) { + lines = append(lines, name+"="+inputs[name]) + } + return strings.Join(lines, "\n") + "\n" +} + +func scheduleInputsDetail(inputs map[string]string) string { + parts := make([]string, 0, len(inputs)) + for _, name := range sortedInputNames(inputs) { + parts = append(parts, name+"="+inputs[name]) + } + return strings.Join(parts, ",") +} + +func sortedInputNames(inputs map[string]string) []string { + names := make([]string, 0, len(inputs)) + for name := range inputs { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/internal/engine/schedule_run_test.go b/internal/engine/schedule_run_test.go new file mode 100644 index 00000000..9a7d0ae1 --- /dev/null +++ b/internal/engine/schedule_run_test.go @@ -0,0 +1,309 @@ +package engine + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/transport" +) + +func TestScheduleRunWritesInputsJournalsThenStartsAfterReleasingTheLock(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog", "prices"}, Default: "catalog"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "systemctl start"): + return transport.Result{}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + result, err := e.ScheduleRun(context.Background(), "20260905-151200-schedule_run-7c1e", "sync", map[string]string{"SOURCE": "prices"}, false) + if err != nil { + t.Fatalf("schedule run: %v\n%s", err, strings.Join(f.Commands, "\n")) + } + if !result.Started || result.Unit != "ob-sample-sync" || result.Inputs["SOURCE"] != "prices" || result.Operation != "20260905-151200-schedule_run-7c1e" { + t.Fatalf("result = %#v", result) + } + seq := strings.Join(f.Commands, "\n") + inputs := strings.Index(seq, "sync.inputs") + start := strings.Index(seq, "systemctl start --no-block 'ob-sample-sync.service'") + release := strings.LastIndex(seq, "rm -f '/var/lib/ob/sample/lock'") + if inputs < 0 || start < 0 || release < 0 || !(inputs < release && release < start) { + t.Fatalf("expected inputs write, lock release, then start:\n%s", seq) + } + if !strings.Contains(seq, "set -C") { + t.Fatalf("inputs file was not created with noclobber:\n%s", seq) + } + if written := strings.Join(f.Inputs, "\n"); !strings.Contains(written, "ONEBOX_OPERATION=20260905-151200-schedule_run-7c1e\nSOURCE=prices\n") { + t.Fatalf("inputs file content is wrong:\n%s", written) + } + for _, want := range []string{ + `"phase":"schedule-run","event":"start"`, + `"phase":"schedule-run","event":"finish","status":"ok"`, + `"target":"sync"`, + `inputs: SOURCE=prices`, + } { + if !strings.Contains(seq, want) { + t.Fatalf("journal is missing %q:\n%s", want, seq) + } + } + // The start is claimed under the lock; the finish records how the request + // ended, so it comes after the unit was actually started. A journal is per + // operation id, so no other operation appends to this file meanwhile. + if started := strings.Index(seq, `"phase":"schedule-run","event":"start"`); started < 0 || started > release { + t.Fatalf("journal start was not written under the lock:\n%s", seq) + } + if finish := strings.LastIndex(seq, `"phase":"schedule-run","event":"finish","status":"ok"`); finish < start { + t.Fatalf("journal finish was written before the unit was started:\n%s", seq) + } +} + +func TestScheduleRunRefusals(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog"}, Default: "catalog"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + cfg.Workloads["prune"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "destructive", + Schedule: &app.JobSchedule{Cron: "0 3 * * *", Timezone: "UTC", Timeout: "1h"}, + } + active := false + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "command -v flock") { + return transport.Result{Stdout: "ok\n"}, true + } + if strings.Contains(cmd, "systemctl is-active") { + if active { + return transport.Result{Stdout: "activating\n"}, true + } + return transport.Result{Stdout: "inactive\n"}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + ctx := context.Background() + if _, err := e.ScheduleRun(ctx, "op", "prune", nil, false); err == nil || !strings.Contains(err.Error(), "ob job run") { + t.Fatalf("destructive job accepted: %v", err) + } + if _, err := e.ScheduleRun(ctx, "op", "sync", map[string]string{"SOURCE": "reviews"}, false); err == nil { + t.Fatal("undeclared value accepted") + } + if _, err := e.ScheduleRun(ctx, "op", "web", nil, false); err == nil { + t.Fatal("non-scheduled workload accepted") + } + for _, command := range f.Commands { + if strings.Contains(command, "systemctl start") || strings.Contains(command, ".inputs") { + t.Fatalf("a refused run reached the host: %s", command) + } + } + active = true + if _, err := e.ScheduleRun(ctx, "op", "sync", nil, false); err == nil || !strings.Contains(err.Error(), "running") { + t.Fatalf("active unit not refused: %v", err) + } +} + +func TestScheduleRunWaitReportsTheRecordAndFailsOnAnyOtherOutcome(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + for outcome, wantErr := range map[string]bool{"success": false, "skipped": true, "failure": true} { + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "systemctl start 'ob-sample-sync.service'"): + return transport.Result{}, true + case strings.Contains(cmd, "SYSLOG_IDENTIFIER=ob-run"): + // Newest first: a stale record from an earlier run precedes ours, + // and must not be mistaken for it. + return transport.Result{Stdout: `{"run":"ffffffffffffffffffffffffffffffff","job":"sync","trigger":"timer","operation":"","started_at":"2026-09-05T14:00:01Z","finished_at":"2026-09-05T14:00:02Z","duration_s":1,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}}` + "\n" + + `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"sync","trigger":"manual","operation":"op-` + outcome + `","started_at":"2026-09-05T15:00:01Z","finished_at":"2026-09-05T15:00:02Z","duration_s":1,"attempts":1,"exit_status":0,"outcome":"` + outcome + `","inputs":{}}` + "\n"}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + result, err := e.ScheduleRun(context.Background(), "op-"+outcome, "sync", nil, true) + if result.Record == nil || result.Record.Outcome != outcome || result.Record.Run != "a1b2c3d4e5f60718293a4b5c6d7e8f90" { + t.Fatalf("%s: this operation's record not returned: %#v", outcome, result) + } + if (err != nil) != wantErr { + t.Fatalf("%s: err = %v, wantErr %v", outcome, err, wantErr) + } + if seq := strings.Join(f.Commands, "\n"); strings.Contains(seq, "--no-block") { + t.Fatalf("%s: --wait must block on systemctl start:\n%s", outcome, seq) + } + } +} + +func TestScheduleRunDiscardsItsInputsWhenTheStartFails(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog", "prices"}, Default: "catalog"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "systemctl start"): + return transport.Result{ExitCode: 5, Stderr: "Unit ob-sample-sync.service not found."}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + _, err := e.ScheduleRun(context.Background(), "op-1", "sync", map[string]string{"SOURCE": "prices"}, false) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("start failure was not reported: %v", err) + } + seq := strings.Join(f.Commands, "\n") + if !strings.Contains(seq, "rm -f '/var/lib/ob/sample/schedule/sync.inputs'") { + t.Fatalf("a failed start left the inputs file pending:\n%s", seq) + } +} + +func TestScheduleRunTellsAPendingFileFromAWriteFailure(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + for name, tc := range map[string]struct { + exit int + stderr string + want string + reject string + }{ + "pending": {73, "", "already pending", "cannot write"}, + "read-only": {1, "cat: cannot create: Read-only file system", "Read-only file system", "already pending"}, + } { + t.Run(name, func(t *testing.T) { + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "sync.inputs"): + return transport.Result{ExitCode: tc.exit, Stderr: tc.stderr}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + _, err := e.ScheduleRun(context.Background(), "op", "sync", nil, false) + if err == nil || !strings.Contains(err.Error(), tc.want) || strings.Contains(err.Error(), tc.reject) { + t.Fatalf("err = %v, want %q and not %q", err, tc.want, tc.reject) + } + for _, command := range f.Commands { + if strings.Contains(command, "systemctl start") { + t.Fatalf("a failed write still started the unit: %s", command) + } + } + }) + } +} + +// Without TRIGGER_UNIT the next timer firing would read the file this run +// left, so the manual path is refused rather than being made ambiguous. +func TestScheduleRunRefusesAHostThatCannotTellTheTriggerApart(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "systemctl --version"): + return transport.Result{Stdout: "systemd 249 (249.11-0ubuntu3)\n"}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + _, err := e.ScheduleRun(context.Background(), "op", "sync", nil, false) + if err == nil || !strings.Contains(err.Error(), "systemd 252") { + t.Fatalf("err = %v, want a refusal naming the requirement", err) + } + for _, command := range f.Commands { + if strings.Contains(command, ".inputs") || strings.Contains(command, "systemctl start") { + t.Fatalf("the refused run still touched the host: %s", command) + } + } +} + +// The journal has to be able to say the request failed. Writing the finish +// as ok before the unit is even started left ob audit reporting "started" for +// a run that never began. +func TestScheduleRunJournalsAFailedRequestAsFailed(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl --version"): + return transport.Result{Stdout: "systemd 255 (255.4-1ubuntu8)\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "systemctl start"): + return transport.Result{ExitCode: 5, Stderr: "Unit ob-sample-sync.service not found."}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + if _, err := e.ScheduleRun(context.Background(), "op-1", "sync", nil, false); err == nil { + t.Fatal("a failed start was reported as success") + } + seq := strings.Join(f.Commands, "\n") + if !strings.Contains(seq, `"phase":"schedule-run","event":"finish","status":"fail"`) { + t.Fatalf("the journal calls a failed request started:\n%s", seq) + } + // The journal redacts a failure's detail on purpose, so the record says + // that the request failed and where to look, not what the host said. + if !strings.Contains(seq, `"error_code":"execution_failed"`) { + t.Fatalf("the failed finish carries no error code:\n%s", seq) + } + if strings.Contains(seq, `"phase":"schedule-run","event":"finish","status":"ok"`) { + t.Fatalf("a failed request also journaled a success:\n%s", seq) + } +} diff --git a/internal/engine/schedule_status.go b/internal/engine/schedule_status.go index f210113d..dac46671 100644 --- a/internal/engine/schedule_status.go +++ b/internal/engine/schedule_status.go @@ -10,31 +10,49 @@ import ( "github.com/labstack/onebox/internal/release" ) -// StatusSchedule is the host-observed state of one declared scheduled job. -// systemd keeps Result after a oneshot exits, so a failed or timed-out run stays -// visible until a later successful run clears it. +// StatusSchedule is the host-observed state of one declared scheduled job. The +// verdict comes from the run records the notifier writes to the journal: +// outcome, attempts, duration, and how many firings in a row have failed or +// been skipped. systemd contributes the timer's state and next elapse and +// whether a run is in progress. type StatusSchedule struct { - Name string `json:"name"` - Unit string `json:"unit"` - TimerState string `json:"timer_state"` - Running bool `json:"running"` - DeployLock string `json:"deploy_lock"` - Timeout string `json:"timeout"` - PinnedRelease string `json:"pinned_release,omitempty"` - StartedAt string `json:"started_at,omitempty"` - LastResult string `json:"last_result"` - LastExitStatus int `json:"last_exit_status,omitempty"` - Diverged bool `json:"diverged"` - Issues []string `json:"issues,omitempty"` + Name string `json:"name"` + Unit string `json:"unit"` + TimerState string `json:"timer_state"` + Running bool `json:"running"` + DeployLock string `json:"deploy_lock"` + Timeout string `json:"timeout"` + PinnedRelease string `json:"pinned_release,omitempty"` + StartedAt string `json:"started_at,omitempty"` + Diverged bool `json:"diverged"` + Issues []string `json:"issues,omitempty"` + + NextRun string `json:"next_run,omitempty"` + Attempt int `json:"attempt,omitempty"` + LastOutcome string `json:"last_outcome,omitempty"` + LastReason string `json:"last_reason,omitempty"` + LastDurationSeconds int `json:"last_duration_s,omitempty"` + LastAttempts int `json:"last_attempts,omitempty"` + ConsecutiveFailures int `json:"consecutive_failures,omitempty"` + ConsecutiveSkips int `json:"consecutive_skips,omitempty"` + // JournalPersistent is false when the host keeps its journal in memory, so + // the records above only reach back to the last boot. + JournalPersistent bool `json:"journal_persistent"` } +// skipStreakIssue is how many firings in a row may be skipped before status +// says so. One skip is timing; a streak is a job that never runs, which is the +// failure mode a skipped record exists to expose. +const skipStreakIssue = 3 + type scheduleUnitObservation struct { loadState string activeState string - result string - exitStatus int release string startedAt string + attempt string + next string + history []ScheduleRunRecord } func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) { @@ -46,16 +64,24 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) return []StatusSchedule{}, nil } - var commands []string + commands := []string{ + "printf '%s\\n' '@@journal'", + "if [ -d /var/log/journal ]; then echo persistent; else echo volatile; fi", + } for _, job := range jobs { unit := e.names().ScheduledJobUnit(job.Name) commands = append(commands, "printf '%s\\n' "+q("@@"+job.Name+":service"), - "systemctl show "+q(unit+".service")+" --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus", + "systemctl show "+q(unit+".service")+" --no-pager --property=LoadState --property=ActiveState", "printf '%s\\n' "+q("@@"+job.Name+":timer"), - "systemctl show "+q(unit+".timer")+" --no-pager --property=LoadState --property=ActiveState", + "systemctl show "+q(unit+".timer")+" --no-pager --property=LoadState --property=ActiveState --property=NextElapseUSecRealtime", "printf '%s\\n' "+q("@@"+job.Name+":run"), "cat "+q(e.names().ScheduledJobRunState(job.Name))+" 2>/dev/null || true", + "printf '%s\\n' "+q("@@"+job.Name+":history"), + // Status degrades rather than fails: an unreadable journal costs + // this section its records, not the whole report. `ob schedule + // history` is the command that says why the read failed. + scheduleHistoryCommand(unit, 20)+" 2>/dev/null || true", ) } res, err := e.T.Run(ctx, strings.Join(commands, "\n")) @@ -67,31 +93,48 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) } observed := map[string]map[string]scheduleUnitObservation{} + journalPersistent := true name, kind := "", "" values := map[string]string{} + var raw []string flush := func() { if name == "" || kind == "" { return } - exit, _ := strconv.Atoi(values["ExecMainStatus"]) if observed[name] == nil { observed[name] = map[string]scheduleUnitObservation{} } observed[name][kind] = scheduleUnitObservation{ loadState: values["LoadState"], activeState: values["ActiveState"], - result: values["Result"], exitStatus: exit, - release: values["release"], startedAt: values["started_at"], + release: values["release"], startedAt: values["started_at"], attempt: values["attempt"], + next: values["NextElapseUSecRealtime"], + history: parseScheduleRunRecords(strings.Join(raw, "\n"), name), } values = map[string]string{} + raw = nil } for _, line := range strings.Split(res.Stdout, "\n") { line = strings.TrimSpace(line) if strings.HasPrefix(line, "@@") { flush() marker := strings.TrimPrefix(line, "@@") + if marker == "journal" { + name, kind = "", "" + continue + } name, kind, _ = strings.Cut(marker, ":") continue } + if name == "" { + if line == "volatile" { + journalPersistent = false + } + continue + } + if kind == "history" { + raw = append(raw, line) + continue + } if key, value, ok := strings.Cut(line, "="); ok { values[key] = value } @@ -104,18 +147,51 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) service := observed[job.Name]["service"] timer := observed[job.Name]["timer"] run := observed[job.Name]["run"] + records := observed[job.Name]["history"].history status := StatusSchedule{ Name: job.Name, Unit: unit, TimerState: timer.activeState, Running: service.activeState == "activating", DeployLock: job.DeployLock, Timeout: job.Timeout, - LastResult: service.result, LastExitStatus: service.exitStatus, + NextRun: timer.next, JournalPersistent: journalPersistent, + } + if status.Running { + status.Attempt, _ = strconv.Atoi(run.attempt) + if job.DeployLock == "pinned" { + _, timeErr := time.Parse(time.RFC3339, run.startedAt) + if !release.IsID(run.release) || timeErr != nil { + status.Issues = append(status.Issues, "running pinned job state is unavailable or invalid") + } else { + status.PinnedRelease = run.release + status.StartedAt = run.startedAt + } + } } - if status.Running && job.DeployLock == "pinned" { - _, timeErr := time.Parse(time.RFC3339, run.startedAt) - if !release.IsID(run.release) || timeErr != nil { - status.Issues = append(status.Issues, "running pinned job state is unavailable or invalid") - } else { - status.PinnedRelease = run.release - status.StartedAt = run.startedAt + // lastRun is the newest record that actually ran: the one whose + // outcome is the job's standing verdict. A skip is news about timing, + // and it must not clear a failure that nothing has fixed yet. + var lastRun *ScheduleRunRecord + if len(records) > 0 { + last := records[0] + status.LastOutcome = last.Outcome + status.LastReason = last.Reason + status.LastDurationSeconds = last.DurationSeconds + status.LastAttempts = last.Attempts + for _, record := range records { + if record.Outcome != "skipped" { + break + } + status.ConsecutiveSkips++ + } + for i, record := range records { + if record.Outcome == "skipped" { + continue + } + if lastRun == nil { + lastRun = &records[i] + } + if record.Outcome != "failure" && record.Outcome != "timeout" { + break + } + status.ConsecutiveFailures++ } } if timer.loadState != "loaded" || timer.activeState != "active" { @@ -124,8 +200,21 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) if service.loadState != "loaded" { status.Issues = append(status.Issues, "service unit is not loaded") } - if service.result != "" && service.result != "success" { - status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %d)", service.result, service.exitStatus)) + // The record is the verdict: the newest run that actually happened is + // the one that counts, and so is a job that keeps being skipped. + if lastRun != nil && (lastRun.Outcome == "failure" || lastRun.Outcome == "timeout") { + exit := "?" + if lastRun.ExitStatus != nil { + exit = strconv.Itoa(*lastRun.ExitStatus) + } + issue := fmt.Sprintf("last run failed: %s (exit %s)", lastRun.Outcome, exit) + if status.LastOutcome == "skipped" { + issue += ", and nothing has run since" + } + status.Issues = append(status.Issues, issue) + } + if status.ConsecutiveSkips >= skipStreakIssue { + status.Issues = append(status.Issues, fmt.Sprintf("skipped %d firings in a row: %s", status.ConsecutiveSkips, status.LastReason)) } status.Diverged = len(status.Issues) > 0 statuses = append(statuses, status) diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index 13b35fac..82be9a1f 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -3,9 +3,11 @@ package engine import ( "bytes" "context" + "encoding/json" "os" "os/exec" "path/filepath" + "regexp" "runtime" "strings" "testing" @@ -178,7 +180,7 @@ func TestScheduledJobUnitContract(t *testing.T) { Calendar: "*-*-* 02:00:00", Timeout: "45m", CatchUp: false, DeployLock: "exclusive", } names := app.Names{App: "sample", BasePath: "/var/lib/ob"} - runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil) + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) service := scheduleServiceUnit("sample", job, "/etc/systemd/system/ob-sample-nightly.run", "/etc/systemd/system/ob-sample-nightly.notify") @@ -186,16 +188,16 @@ func TestScheduledJobUnitContract(t *testing.T) { for _, want := range []string{ "exec 9>'/var/lib/ob/sample/schedule/nightly.lock'", - "flock --exclusive --nonblock --conflict-exit-code 75 9", + "flock --exclusive --nonblock 9 || stand_aside", "exec 8>'/var/lib/ob/sample/schedule.lock'", - "flock --exclusive --nonblock --conflict-exit-code 75 8", + "flock --exclusive --nonblock 8 || skip", "/var/lib/ob/sample/lock", "application operation holds the deploy lock", "docker compose", "--project-directory", "/var/lib/ob/sample/current", "compose.yaml", - "run --rm --no-deps --name 'sample-nightly-1'", + `run --rm --no-deps "$@" --name 'sample-nightly-1'`, "docker rm -f 'sample-nightly-1'", "nightly", } { @@ -240,11 +242,11 @@ func TestPinnedScheduledJobRunnerLeasesImmutableRelease(t *testing.T) { runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", []app.EnvFile{ {File: "config/runtime.env"}, {File: "secrets/runtime.env", Provider: "sops"}, - }) + }, 10*time.Minute, true) for _, want := range []string{ "exec 9>'/var/lib/ob/sample/schedule/refresh.lock'", - "flock --exclusive --nonblock --conflict-exit-code 75 9", + "flock --exclusive --nonblock 9 || stand_aside", "exec 8>'/var/lib/ob/sample/schedule.lock'", "release_dir=$(readlink -f '/var/lib/ob/sample/current')", "exec 7>>\"$release_dir/.ob-schedule.lease\"", @@ -256,7 +258,7 @@ func TestPinnedScheduledJobRunnerLeasesImmutableRelease(t *testing.T) { "--project-directory \"$release_dir\"", "-f \"$release_dir\"/'compose.yaml'", "--env-file \"$release_dir\"/'config/runtime.env'", - "run --rm --no-deps --name 'sample-refresh-1' 'refresh'", + `run --rm --no-deps "$@" --name 'sample-refresh-1' 'refresh'`, "docker rm -f 'sample-refresh-1'", } { if !strings.Contains(runner, want) { @@ -340,7 +342,7 @@ func TestPinnedScheduledJobLockProtocol(t *testing.T) { } job := app.ScheduledJob{Name: "refresh", DeployLock: "pinned"} - runner := scheduleRunnerScript("sample", job, names, filepath.Join(names.AppDir(), "lock"), nil) + runner := scheduleRunnerScript("sample", job, names, filepath.Join(names.AppDir(), "lock"), nil, 10*time.Minute, true) runner = strings.ReplaceAll(runner, "/usr/bin/docker", q(stub)) command := exec.CommandContext(ctx, "sh") command.Stdin = strings.NewReader(runner) @@ -426,8 +428,10 @@ func TestPinnedScheduledJobLockProtocol(t *testing.T) { if err != nil || len(leases) != 0 { t.Fatalf("completed release remained leased: leases=%v err=%v", leases, err) } - if _, err := os.Stat(names.ScheduledJobRunState(job.Name)); !os.IsNotExist(err) { - t.Fatalf("runner state survived completion: %v", err) + // The notifier, not the runner, removes the state: it is the evidence + // ExecStopPost turns into the run record. + if _, err := os.Stat(names.ScheduledJobRunState(job.Name)); err != nil { + t.Fatalf("runner removed the state the notifier finalises: %v", err) } } @@ -445,7 +449,7 @@ func TestScheduledJobFailureNotifierUsesConfiguredWebhooks(t *testing.T) { } f := &transport.Fake{TargetName: "root@example.internal"} e := New(cfg, testProject(t), f, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) - script, err := e.scheduleFailureNotifier("nightly") + script, err := e.scheduleNotifier(app.ScheduledJob{Name: "nightly", Notify: []string{"failure", "timeout"}}) if err != nil { t.Fatal(err) } @@ -503,39 +507,6 @@ func TestSyncSchedulesRefusesMissingFlockBeforeInstallingUnits(t *testing.T) { } } -func TestScheduleStatusSurfacesTheLastSystemdFailure(t *testing.T) { - cfg := testConfig() - cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", - Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, - } - f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { - if strings.Contains(cmd, "systemctl show") { - return transport.Result{Stdout: `@@nightly:service -LoadState=loaded -ActiveState=failed -Result=timeout -ExecMainStatus=15 -@@nightly:timer -LoadState=loaded -ActiveState=active -`}, true - } - return transport.Result{}, false - }} - e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) - statuses, err := e.scheduleStatuses(context.Background()) - if err != nil { - t.Fatal(err) - } - if len(statuses) != 1 || !statuses[0].Diverged || statuses[0].LastResult != "timeout" || statuses[0].LastExitStatus != 15 { - t.Fatalf("failed systemd result was not surfaced: %#v", statuses) - } - if !strings.Contains(strings.Join(statuses[0].Issues, "\n"), "last run failed") { - t.Fatalf("failure has no actionable issue: %#v", statuses[0]) - } -} - func TestScheduleStatusReportsRunningPinnedRelease(t *testing.T) { cfg := testConfig() cfg.Workloads["refresh"] = app.Workload{ @@ -822,3 +793,814 @@ func TestAppNamedBackupDoesNotOwnEveryBackupTimer(t *testing.T) { t.Fatalf("app named backup removed another application's timer:\n%s", seq) } } + +func TestScheduledJobRunnersRecordRunStateForTheNotifier(t *testing.T) { + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + for _, tc := range []struct { + name string + job app.ScheduledJob + }{ + {"exclusive", app.ScheduledJob{Name: "nightly", Cron: "0 2 * * *", Timezone: "UTC", Calendar: "*-*-* 02:00:00", Timeout: "45m", DeployLock: "exclusive"}}, + {"pinned", app.ScheduledJob{Name: "nightly", Cron: "0 2 * * *", Timezone: "UTC", Calendar: "*-*-* 02:00:00", Timeout: "45m", DeployLock: "pinned"}}, + } { + t.Run(tc.name, func(t *testing.T) { + runner := scheduleRunnerScript("sample", tc.job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + for _, want := range []string{ + "state='/var/lib/ob/sample/schedule/nightly.state'", + "write_state() {", + "started_epoch=%s", + "trigger=%s", + "write_state 1", + "mv -f \"$tmp\" \"$state\"", + } { + if !strings.Contains(runner, want) { + t.Errorf("%s runner is missing %q:\n%s", tc.name, want, runner) + } + } + if strings.Contains(runner, `rm -f "$state"`) { + t.Errorf("%s runner removes the state the notifier finalises:\n%s", tc.name, runner) + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(runner) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("%s runner is not valid POSIX shell: %v: %s\n%s", tc.name, err, output, runner) + } + }) + } + service := scheduleServiceUnit("sample", app.ScheduledJob{Name: "nightly", Timeout: "45m"}, + "/etc/systemd/system/ob-sample-nightly.run", "/etc/systemd/system/ob-sample-nightly.notify") + if strings.Contains(service, "SuccessExitStatus") { + t.Errorf("a skip is recorded by the runner and exits 0; the unit needs no exit-status remap:\n%s", service) + } +} + +func TestScheduledJobNotifierWritesOneRunRecordToTheJournal(t *testing.T) { + cfg := testConfig() + f := &transport.Fake{TargetName: "root@example.internal"} + e := New(cfg, testProject(t), f, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + script, err := e.scheduleNotifier(app.ScheduledJob{Name: "nightly", Notify: []string{"failure", "timeout"}}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "state='/var/lib/ob/sample/schedule/nightly.state'", + `started_epoch=*) started_epoch=${line#started_epoch=}`, + `rm -f "$state"`, + `result=${SERVICE_RESULT:-success}`, + `status=${EXIT_STATUS:-0}`, + `outcome=timeout`, + `outcome=skipped`, + `outcome=success`, + `outcome=failure`, + `"run":"%s","job":"%s","trigger":"%s","operation":"%s","release":"%s"`, + `"duration_s":%s,"attempts":%s,"exit_status":%s,"outcome":"%s","reason":"%s","inputs":{%s}`, + `"${INVOCATION_ID:-}" 'nightly'`, + `SYSLOG_IDENTIFIER=ob-run\nONEBOX_APP=%s\nONEBOX_UNIT=%s\nONEBOX_JOB=%s`, + `"$record" 'sample' 'ob-sample-nightly' 'nightly' | logger --journald`, + } { + if !strings.Contains(script, want) { + t.Errorf("notifier is missing %q:\n%s", want, script) + } + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(script) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("notifier is not valid POSIX shell: %v: %s\n%s", err, output, script) + } +} + +// runNotifier executes the generated ExecStopPost script with stub logger and +// curl binaries, the way systemd would after a run. It returns the record the +// script wrote, whether the state file survived, and every curl invocation's +// arguments, one per element. +func runNotifier(t *testing.T, job app.ScheduledJob, notifications map[string]app.Notification, state string, env map[string]string) (map[string]any, bool, []string) { + t.Helper() + base := t.TempDir() + if state != "" { + scheduleDir := filepath.Join(base, "sample", "schedule") + if err := os.MkdirAll(scheduleDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(scheduleDir, "nightly.state"), []byte(state), 0o600); err != nil { + t.Fatal(err) + } + } + return runNotifierIn(t, base, job, notifications, env) +} + +// runNotifierIn is runNotifier over a directory the caller has already laid +// out, for the cases whose whole point is which files exist beforehand. +func runNotifierIn(t *testing.T, base string, job app.ScheduledJob, notifications map[string]app.Notification, env map[string]string) (map[string]any, bool, []string) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("POSIX shell required") + } + cfg := testConfig() + cfg.BasePath = base + cfg.Notifications = notifications + e := New(cfg, testProject(t), &transport.Fake{TargetName: "root@example.internal"}, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + script, err := e.scheduleNotifier(job) + if err != nil { + t.Fatal(err) + } + scheduleDir := filepath.Join(base, "sample", "schedule") + if err := os.MkdirAll(scheduleDir, 0o700); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(scheduleDir, "nightly.state") + bin := t.TempDir() + record := filepath.Join(bin, "record.jsonl") + // The stub checks the structured fields the history query relies on and + // keeps only the MESSAGE line, as `journalctl -o cat` would show it. + stub := "#!/bin/sh\n[ \"$1\" = --journald ] || exit 9\n" + + "fields=$(cat)\n" + + "printf '%s\\n' \"$fields\" | grep -q '^SYSLOG_IDENTIFIER=ob-run$' || exit 8\n" + + "printf '%s\\n' \"$fields\" | grep -q '^ONEBOX_UNIT=ob-sample-nightly$' || exit 7\n" + + "printf '%s\\n' \"$fields\" | grep -q '^ONEBOX_JOB=nightly$' || exit 6\n" + + "printf '%s\\n' \"$fields\" | sed -n 's/^MESSAGE=//p' >>" + record + "\n" + if err := os.WriteFile(filepath.Join(bin, "logger"), []byte(stub), 0o755); err != nil { + t.Fatal(err) + } + sent := filepath.Join(bin, "curl.args") + // Sends run in the background concurrently, so each stub call writes its + // whole argument list in one printf, keeping calls from interleaving. + curl := "#!/bin/sh\nprintf '%s\\n' \"$@\" -- >>" + sent + "\n" + if err := os.WriteFile(filepath.Join(bin, "curl"), []byte(curl), 0o755); err != nil { + t.Fatal(err) + } + command := exec.CommandContext(context.Background(), "sh", "-s") + command.Stdin = strings.NewReader(script) + command.Env = append([]string{"PATH=" + bin + ":" + os.Getenv("PATH")}, "HOME="+base) + for k, v := range env { + command.Env = append(command.Env, k+"="+v) + } + if out, err := command.CombinedOutput(); err != nil { + t.Fatalf("notifier exited non-zero: %v\n%s\n%s", err, out, script) + } + body, err := os.ReadFile(record) + if err != nil { + t.Fatalf("notifier wrote no record: %v", err) + } + lines := strings.Split(strings.TrimSpace(string(body)), "\n") + if len(lines) != 1 { + t.Fatalf("notifier wrote %d records, want 1:\n%s", len(lines), body) + } + var decoded map[string]any + if err := json.Unmarshal([]byte(lines[0]), &decoded); err != nil { + t.Fatalf("record is not JSON: %v\n%s", err, lines[0]) + } + _, stateErr := os.Stat(statePath) + var sends []string + if args, err := os.ReadFile(sent); err == nil { + sends = strings.Split(strings.TrimSpace(string(args)), "\n") + } + return decoded, stateErr == nil, sends +} + +func TestScheduledJobNotifierRecordsEachOutcomeAndRemovesState(t *testing.T) { + state := "release=20260905-140000-ab12cd\nstarted_at=2026-09-05T15:00:01Z\nstarted_epoch=1\ntrigger=timer\noperation=\nattempt=2\ninputs=\n" + for name, tc := range map[string]struct { + state string + env map[string]string + outcome string + exit any + attempt float64 + }{ + "success": {state, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "a1b2"}, "success", float64(0), 2}, + "failure": {state, map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "1"}, "failure", float64(1), 2}, + "timeout": {state, map[string]string{"SERVICE_RESULT": "timeout", "EXIT_STATUS": "TERM"}, "timeout", nil, 2}, + "skipped": {"skipped=another run of this job is still in progress\noperation=\ninputs=\n", map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "TRIGGER_UNIT": "ob-sample-nightly.timer"}, "skipped", float64(0), 0}, + "job exits 75": {state, map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "75"}, "failure", float64(75), 2}, + "no state": {"", map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "3"}, "failure", float64(3), 0}, + } { + t.Run(name, func(t *testing.T) { + record, stateLeft, sends := runNotifier(t, app.ScheduledJob{Name: "nightly", Notify: []string{"failure", "timeout"}}, nil, tc.state, tc.env) + if len(sends) != 0 { + t.Fatalf("no webhook is configured, yet curl ran: %v", sends) + } + if record["outcome"] != tc.outcome || record["exit_status"] != tc.exit || record["attempts"] != tc.attempt { + t.Fatalf("record = %#v", record) + } + if record["job"] != "nightly" || record["run"] != tc.env["INVOCATION_ID"] { + t.Fatalf("identity fields wrong: %#v", record) + } + if tc.state == state && (record["release"] != "20260905-140000-ab12cd" || record["trigger"] != "timer" || record["duration_s"].(float64) < 1) { + t.Fatalf("state fields not carried: %#v", record) + } + if name == "skipped" && (record["trigger"] != "timer" || record["reason"] != "another run of this job is still in progress") { + t.Fatalf("skip was not recorded with its trigger and reason: %#v", record) + } + if stateLeft { + t.Fatal("state file survived the notifier") + } + }) + } +} + +func TestScheduleStatusPrefersTheRunRecordOverSystemdResult(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: `@@journal +persistent +@@nightly:service +LoadState=loaded +ActiveState=inactive +Result=success +ExecMainStatus=75 +@@nightly:timer +LoadState=loaded +ActiveState=active +NextElapseUSecRealtime=Sat 2026-09-06 02:00:00 UTC +@@nightly:run +@@nightly:history +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","release":"r1","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:02Z","duration_s":1,"attempts":0,"exit_status":75,"outcome":"skipped","inputs":{}} +{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"timer","release":"r1","started_at":"2026-09-04T02:00:01Z","finished_at":"2026-09-04T02:05:02Z","duration_s":301,"attempts":3,"exit_status":1,"outcome":"failure","inputs":{}} +{"run":"c3d4e5f60718293a4b5c6d7e8f901234","job":"nightly","trigger":"timer","release":"r1","started_at":"2026-09-03T02:00:01Z","finished_at":"2026-09-03T02:01:02Z","duration_s":61,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}} +`}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + got := statuses[0] + if got.LastOutcome != "skipped" || got.NextRun != "Sat 2026-09-06 02:00:00 UTC" || !got.JournalPersistent { + t.Fatalf("record fields not surfaced: %#v", got) + } + // The newest record is a skip: it is not itself a failure, and it does not + // clear the failure before it, which is still the job's standing verdict. + if got.ConsecutiveFailures != 1 || !got.Diverged { + t.Fatalf("a skip cleared the failure behind it: %#v", got) + } + if got.LastAttempts != 0 || got.LastDurationSeconds != 1 { + t.Fatalf("last run detail not surfaced: %#v", got) + } +} + +func TestScheduleStatusCountsConsecutiveFailuresFromRecords(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: `@@journal +volatile +@@nightly:service +LoadState=loaded +ActiveState=failed +Result=exit-code +ExecMainStatus=1 +@@nightly:timer +LoadState=loaded +ActiveState=active +@@nightly:run +@@nightly:history +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:02Z","duration_s":1,"attempts":2,"exit_status":1,"outcome":"failure","inputs":{}} +{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"timer","started_at":"2026-09-04T02:00:01Z","finished_at":"2026-09-04T02:05:02Z","duration_s":301,"attempts":2,"exit_status":null,"outcome":"timeout","inputs":{}} +{"run":"c3d4e5f60718293a4b5c6d7e8f901234","job":"nightly","trigger":"timer","started_at":"2026-09-03T02:00:01Z","finished_at":"2026-09-03T02:01:02Z","duration_s":61,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}} +`}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + got := statuses[0] + if got.ConsecutiveFailures != 2 || !got.Diverged || got.JournalPersistent { + t.Fatalf("failures not counted: %#v", got) + } + if !strings.Contains(strings.Join(got.Issues, "; "), "last run failed: failure (exit 1)") { + t.Fatalf("issue does not name the outcome: %#v", got.Issues) + } +} + +func TestScheduledJobRunnerRetriesWithCappedDoublingBackoff(t *testing.T) { + job := app.ScheduledJob{Name: "nightly", Timeout: "45m", DeployLock: "exclusive", + RetryAttempts: 3, RetryBackoff: 30 * time.Second, RetryMaxBackoff: 10 * time.Minute} + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + for _, want := range []string{ + "max_attempts=3", "backoff=30", "max_backoff=600", + "attempt=1", "while :; do", "write_state \"$attempt\"", + "status=0", "|| status=$?", "[ \"$status\" -eq 0 ] && exit 0", + "if [ \"$attempt\" -ge \"$max_attempts\" ]; then exit \"$status\"; fi", + "sleep \"$backoff\"", "backoff=$((backoff * 2))", "attempt=$((attempt + 1))", + } { + if !strings.Contains(runner, want) { + t.Errorf("runner is missing %q:\n%s", want, runner) + } + } + single := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + if strings.Contains(single, "max_attempts=") { + t.Errorf("a single-attempt job must not carry a retry loop:\n%s", single) + } + pinned := scheduleRunnerScript("sample", app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "pinned", RetryAttempts: 2, RetryBackoff: time.Second, RetryMaxBackoff: time.Minute}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + if !strings.Contains(pinned, "max_attempts=") || strings.Index(pinned, "flock --unlock 8") > strings.Index(pinned, "max_attempts=") { + t.Errorf("pinned runner must release the schedule mutex before its attempt loop:\n%s", pinned) + } + for _, script := range []string{runner, single, pinned} { + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(script) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("runner is not valid POSIX shell: %v: %s\n%s", err, output, script) + } + } +} + +func TestScheduledJobNotifierSelectsBodiesByOutcome(t *testing.T) { + cfg := testConfig() + cfg.Notifications = map[string]app.Notification{ + "ops": {Webhook: "https://hooks.example.com/ops", On: []string{"success", "failure"}, Format: "json"}, + } + f := &transport.Fake{TargetName: "root@example.internal"} + e := New(cfg, testProject(t), f, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + script, err := e.scheduleNotifier(app.ScheduledJob{Name: "nightly", Notify: []string{"success", "failure", "skipped"}}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `case " success failure skipped " in *" $outcome "*) ;; *) exit 0 ;; esac`, + `"status":"ok"`, + `"status":"fail"`, + `if [ "$outcome" = success ]; then`, + `"deploy_id":"'"${INVOCATION_ID:-}"'"`, + } { + if !strings.Contains(script, want) { + t.Errorf("notifier is missing %q:\n%s", want, script) + } + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(script) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("notifier is not valid POSIX shell: %v: %s\n%s", err, output, script) + } +} + +func TestScheduledJobNotifierSendsOnlySelectedOutcomesWithTheRunID(t *testing.T) { + state := "release=r1\nstarted_at=2026-09-05T15:00:01Z\nstarted_epoch=1\ntrigger=timer\noperation=\nattempt=2\ninputs=\n" + webhooks := map[string]app.Notification{ + "ops": {Webhook: "https://hooks.example.com/ops", On: []string{"success", "failure"}, Format: "json"}, + "chat": {Webhook: "https://hooks.example.com/chat", On: []string{"failure"}, Format: "text"}, + } + for name, tc := range map[string]struct { + notify []string + env map[string]string + want int + status string + }{ + "failure selected": {[]string{"failure", "timeout"}, map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "1", "INVOCATION_ID": "abc123"}, 2, "fail"}, + "success not selected": {[]string{"failure", "timeout"}, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "abc123"}, 0, ""}, + "success selected": {[]string{"success"}, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "abc123"}, 1, "ok"}, + "skipped selected": {[]string{"skipped"}, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "abc123"}, 2, "fail"}, + "timeout not selected": {[]string{"failure"}, map[string]string{"SERVICE_RESULT": "timeout", "EXIT_STATUS": "TERM", "INVOCATION_ID": "abc123"}, 0, ""}, + } { + t.Run(name, func(t *testing.T) { + runState := state + if name == "skipped selected" { + runState = "skipped=an application operation holds the deploy lock\noperation=\ninputs=\n" + } + _, _, sends := runNotifier(t, app.ScheduledJob{Name: "nightly", Notify: tc.notify}, webhooks, runState, tc.env) + calls := 0 + for _, arg := range sends { + if arg == "--" { + calls++ + } + } + if calls != tc.want { + t.Fatalf("curl ran %d time(s), want %d:\n%s", calls, tc.want, strings.Join(sends, "\n")) + } + if tc.want == 0 { + return + } + joined := strings.Join(sends, "\n") + var jsonBody string + for _, arg := range sends { + if strings.HasPrefix(arg, "{") { + jsonBody = arg + } + } + if jsonBody == "" { + t.Fatalf("no JSON body was sent:\n%s", joined) + } + var body map[string]any + if err := json.Unmarshal([]byte(jsonBody), &body); err != nil { + t.Fatalf("sent body is not JSON: %v\n%s", err, jsonBody) + } + if body["status"] != tc.status || body["deploy_id"] != "abc123" || body["verb"] != "scheduled job nightly" { + t.Fatalf("body = %#v", body) + } + if ts, _ := body["ts"].(string); !strings.HasSuffix(ts, "Z") || strings.Contains(ts, "ONEBOX") { + t.Fatalf("timestamp was not filled at send time: %#v", body) + } + if strings.Contains(joined, "ONEBOX_SCHEDULE") { + t.Fatalf("a placeholder leaked into a send:\n%s", joined) + } + }) + } +} + +func TestScheduledJobRunnerConsumesManualInputsWithoutShellInterpolation(t *testing.T) { + job := app.ScheduledJob{Name: "sync", Timeout: "45m", DeployLock: "pinned", RetryAttempts: 1, + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog"}, Default: "catalog"}}} + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + for _, want := range []string{ + "inputs_file='/var/lib/ob/sample/schedule/sync.inputs'", + `if [ -z "${TRIGGER_UNIT:-}" ] && [ -f "$inputs_file" ]; then`, + `while IFS= read -r line || [ -n "$line" ]; do`, + `ONEBOX_OPERATION=*) operation=${line#ONEBOX_OPERATION=} ;;`, + `[A-Z]*=*) set -- "$@" -e "$line"`, + `rm -f "$inputs_file"`, + `run --rm --no-deps "$@" --name 'sample-sync-1' 'sync'`, + } { + if !strings.Contains(runner, want) { + t.Errorf("runner is missing %q:\n%s", want, runner) + } + } + if strings.Contains(runner, ". \"$inputs_file\"") || strings.Contains(runner, "eval") { + t.Fatalf("runner evaluates the inputs file as shell:\n%s", runner) + } + command := exec.CommandContext(context.Background(), "sh", "-n") + command.Stdin = strings.NewReader(runner) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("runner is not valid POSIX shell: %v: %s\n%s", err, output, runner) + } + // The consume block precedes the locks so a skipped manual run cannot + // leave its inputs for the next timer firing. + if strings.Index(runner, "inputs_file=") > strings.Index(runner, "flock --exclusive --nonblock 9") { + t.Fatalf("inputs are consumed after the lock:\n%s", runner) + } + exclusive := scheduleRunnerScript("sample", app.ScheduledJob{Name: "sync", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + if !strings.Contains(exclusive, "inputs_file=") || !strings.Contains(exclusive, `run --rm --no-deps "$@" --name`) { + t.Fatalf("exclusive runner does not consume inputs:\n%s", exclusive) + } +} + +func TestSyncSchedulesRequireSystemd252ForEveryScheduledJob(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"a"}, Default: "a"}}, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + for version, wantErr := range map[string]bool{"systemd 249 (249.11-0ubuntu3)\n": true, "systemd 255 (255.4-1ubuntu8)\n": false, "": true} { + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "list-unit-files"): + return transport.Result{}, true + case strings.Contains(cmd, "systemd-analyze calendar"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl --version"): + return transport.Result{Stdout: version}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + err := e.SyncSchedules(context.Background()) + if wantErr && (err == nil || !strings.Contains(err.Error(), "systemd 252")) { + t.Fatalf("version %q was accepted for a scheduled job: %v", version, err) + } + if !wantErr && err != nil { + t.Fatalf("version %q was refused: %v", version, err) + } + } +} + +// The inputs block is the one place operator text meets the runner, so it is +// executed rather than only inspected: values with spaces and equals signs +// must arrive as single -e arguments, the metadata line must never become an +// argument, and the file must be gone afterwards. +func TestScheduleInputsLinesParseTheFileIntoArguments(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX shell required") + } + dir := t.TempDir() + inputs := filepath.Join(dir, "sync.inputs") + body := "ONEBOX_OPERATION=20260905-151200-schedule_run-7c1e\nSOURCE=prices and more\nSINCE=2026-09-01=ish\n" + if err := os.WriteFile(inputs, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + script := strings.Join(append(append([]string{"#!/bin/sh", "set -eu"}, scheduleInputsLines(inputs)...), + `printf 'argc=%s\n' "$#"`, + `for arg in "$@"; do printf 'arg=%s\n' "$arg"; done`, + `printf 'operation=%s\n' "$operation"`, + `printf 'json=%s\n' "$inputs_json"`, + ), "\n") + for _, trigger := range []string{"", "ob-sample-sync.timer"} { + command := exec.CommandContext(context.Background(), "sh", "-s") + command.Stdin = strings.NewReader(script) + command.Env = []string{"PATH=" + os.Getenv("PATH")} + if trigger != "" { + command.Env = append(command.Env, "TRIGGER_UNIT="+trigger) + } + out, err := command.CombinedOutput() + if err != nil { + t.Fatalf("trigger %q: %v\n%s", trigger, err, out) + } + got := string(out) + if trigger != "" { + if !strings.Contains(got, "argc=0\n") || !strings.Contains(got, "operation=\n") { + t.Fatalf("a timer activation read the inputs file:\n%s", got) + } + if _, err := os.Stat(inputs); err != nil { + t.Fatalf("a timer activation removed the inputs file: %v", err) + } + continue + } + for _, want := range []string{ + "argc=4\n", "arg=-e\narg=SOURCE=prices and more\n", "arg=-e\narg=SINCE=2026-09-01=ish\n", + "operation=20260905-151200-schedule_run-7c1e\n", + `json="SOURCE":"prices and more","SINCE":"2026-09-01=ish"` + "\n", + } { + if !strings.Contains(got, want) { + t.Fatalf("manual activation output is missing %q:\n%s", want, got) + } + } + if _, err := os.Stat(inputs); err == nil { + t.Fatal("the inputs file survived a manual activation") + } + if err := os.WriteFile(inputs, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } +} + +func TestScheduleStatusRaisesAnIssueForASkipStreak(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + skip := `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:01Z","duration_s":0,"attempts":0,"exit_status":0,"outcome":"skipped","reason":"an application operation holds the deploy lock","inputs":{}}` + success := `{"run":"c3d4e5f60718293a4b5c6d7e8f901234","job":"nightly","trigger":"timer","started_at":"2026-09-03T02:00:01Z","finished_at":"2026-09-03T02:01:02Z","duration_s":61,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}}` + for name, tc := range map[string]struct { + history string + skips int + issue bool + }{ + "two skips": {skip + "\n" + skip + "\n" + success, 2, false}, + "three skips": {skip + "\n" + skip + "\n" + skip + "\n" + success, 3, true}, + } { + t.Run(name, func(t *testing.T) { + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: "@@journal\npersistent\n@@nightly:service\nLoadState=loaded\nActiveState=inactive\n@@nightly:timer\nLoadState=loaded\nActiveState=active\n@@nightly:run\n@@nightly:history\n" + tc.history + "\n"}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + got := statuses[0] + if got.ConsecutiveSkips != tc.skips || got.LastOutcome != "skipped" || got.LastReason == "" { + t.Fatalf("skips not counted: %#v", got) + } + if got.Diverged != tc.issue { + t.Fatalf("issue = %v, want %v: %#v", got.Diverged, tc.issue, got.Issues) + } + if tc.issue && !strings.Contains(strings.Join(got.Issues, "; "), "skipped 3 firings in a row: an application operation holds the deploy lock") { + t.Fatalf("issue does not name the streak and reason: %#v", got.Issues) + } + }) + } +} + +// A skip is news about timing. It must not clear a failure that nothing has +// fixed: the newest record that actually ran is the standing verdict. +func TestScheduleStatusKeepsAFailureVisibleBehindASkip(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + history := `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:01Z","duration_s":0,"attempts":0,"exit_status":0,"outcome":"skipped","reason":"an application operation holds the deploy lock","inputs":{}} +{"run":"b2c3d4e5f60718293a4b5c6d7e8f9012","job":"nightly","trigger":"timer","started_at":"2026-09-04T02:00:01Z","finished_at":"2026-09-04T02:05:02Z","duration_s":301,"attempts":2,"exit_status":9,"outcome":"failure","inputs":{}} +{"run":"c3d4e5f60718293a4b5c6d7e8f901234","job":"nightly","trigger":"timer","started_at":"2026-09-03T02:00:01Z","finished_at":"2026-09-03T02:01:02Z","duration_s":61,"attempts":1,"exit_status":0,"outcome":"success","inputs":{}}` + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: "@@journal\npersistent\n@@nightly:service\nLoadState=loaded\nActiveState=inactive\n@@nightly:timer\nLoadState=loaded\nActiveState=active\n@@nightly:run\n@@nightly:history\n" + history + "\n"}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + got := statuses[0] + if !got.Diverged || got.ConsecutiveFailures != 1 || got.LastOutcome != "skipped" { + t.Fatalf("a skip cleared the failure behind it: %#v", got) + } + if issues := strings.Join(got.Issues, "; "); !strings.Contains(issues, "last run failed: failure (exit 9)") || + !strings.Contains(issues, "nothing has run since") { + t.Fatalf("issue does not name the failure the skip hid: %#v", got.Issues) + } +} + +// An unreadable journal costs status its records, not the whole report. +func TestScheduleStatusDegradesWhenTheJournalCannotBeRead(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + if !strings.Contains(cmd, "2>/dev/null || true") { + return transport.Result{ExitCode: 1, Stderr: "Failed to open journal"}, true + } + return transport.Result{Stdout: "@@journal\npersistent\n@@nightly:service\nLoadState=loaded\nActiveState=inactive\n@@nightly:timer\nLoadState=loaded\nActiveState=active\n@@nightly:run\n@@nightly:history\n"}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatalf("an unreadable journal broke the whole status read: %v", err) + } + if got := statuses[0]; got.LastOutcome != "" || got.Diverged { + t.Fatalf("missing records must read as unknown, not as failure: %#v", got) + } +} + +// A firing that cannot take the job lock must leave the running job's state +// alone: that file is the evidence its own notifier turns into the record. +func TestScheduledJobRunnerDoesNotClobberARunningJobsState(t *testing.T) { + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + job := app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + // The note is keyed to this activation, so it cannot be mistaken for the + // state file of the run that holds the lock. + if !strings.Contains(runner, `skip_marker="$state.skip.${INVOCATION_ID:-}"`) || + !strings.Contains(runner, `stand_aside() { umask 077; printf 'skipped=%s\noperation=%s\ninputs=%s\n' "$1" "$operation" "$inputs_json" >"$skip_marker"`) { + t.Fatalf("runner has no lock-less skip note:\n%s", runner) + } + if strings.Contains(runner, `stand_aside`) && strings.Contains(runner, `>"$state"; echo "onebox: skipped`) { + t.Fatalf("the lock-less skip writes the running job's state:\n%s", runner) + } + if !strings.Contains(runner, "flock --exclusive --nonblock 9 || stand_aside 'another run of this job is still in progress'") { + t.Fatalf("a job-lock conflict still writes state:\n%s", runner) + } + // The other two skips hold the job lock, so the state is theirs to write. + for _, want := range []string{ + "flock --exclusive --nonblock 8 || skip 'an application operation is taking its lock'", + "skip 'an application operation holds the deploy lock'", + } { + if !strings.Contains(runner, want) { + t.Fatalf("runner is missing %q:\n%s", want, runner) + } + } +} + +// The container name is fixed, so a corpse from one attempt would fail every +// attempt after it. +func TestScheduledJobRunnerClearsTheContainerBetweenAttempts(t *testing.T) { + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + job := app.ScheduledJob{Name: "nightly", Timeout: "45m", DeployLock: "exclusive", + RetryAttempts: 3, RetryBackoff: time.Second, RetryMaxBackoff: time.Minute} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + loop := runner[strings.Index(runner, "while :; do"):] + if !strings.Contains(loop, "docker rm -f 'sample-nightly-1'") { + t.Fatalf("no cleanup inside the attempt loop:\n%s", loop) + } +} + +// On a systemd without TRIGGER_UNIT the runner cannot see the trigger, and +// says so rather than calling every timer firing an operator's run. +func TestScheduledJobRunnerRecordsAnUnknownTriggerOnAnOlderSystemd(t *testing.T) { + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + job := app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1} + modern := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + older := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, false) + if !strings.Contains(modern, "else trigger=manual; fi") { + t.Fatalf("a host that sets TRIGGER_UNIT must name the operator:\n%s", modern) + } + if !strings.Contains(older, "else trigger=unknown; fi") { + t.Fatalf("a host without TRIGGER_UNIT must not invent a trigger:\n%s", older) + } +} + +// The systemd floor is scoped to what actually needs it. A project that has +// been running scheduled jobs on an older LTS keeps running them. +func TestSyncSchedulesRequiresSystemd252OnlyForInputs(t *testing.T) { + for name, tc := range map[string]struct { + inputs map[string]app.JobInput + wantErr bool + }{ + "plain job": {nil, false}, + "declares inputs": {map[string]app.JobInput{"SOURCE": {Enum: []string{"a"}, Default: "a"}}, true}, + } { + t.Run(name, func(t *testing.T) { + cfg := testConfig() + cfg.Workloads["sync"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", Inputs: tc.inputs, + Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "list-unit-files"): + return transport.Result{}, true + case strings.Contains(cmd, "systemd-analyze calendar"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl --version"): + return transport.Result{Stdout: "systemd 249 (249.11-0ubuntu3)\n"}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + err := e.SyncSchedules(context.Background()) + if tc.wantErr { + if err == nil || !strings.Contains(err.Error(), "systemd 252") { + t.Fatalf("inputs were accepted on systemd 249: %v", err) + } + return + } + if err != nil { + t.Fatalf("a job without inputs was refused on systemd 249: %v", err) + } + if artifacts := strings.Join(f.Inputs, "\n"); !strings.Contains(artifacts, "else trigger=unknown; fi") { + t.Fatalf("the runner claims a trigger the host cannot report:\n%s", artifacts) + } + }) + } +} + +// An activation that stood aside must record itself as skipped without +// touching the state file the running job is still writing. Getting this +// wrong records a run that never happened as a success, and destroys the +// evidence of the one that did. +func TestScheduledJobNotifierReadsAStandAsideNoteAndSpareTheRunningState(t *testing.T) { + base := t.TempDir() + scheduleDir := filepath.Join(base, "sample", "schedule") + if err := os.MkdirAll(scheduleDir, 0o700); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(scheduleDir, "nightly.state") + liveState := "release=r1\nstarted_at=2026-09-05T15:00:01Z\nstarted_epoch=1\ntrigger=timer\noperation=\nattempt=2\ninputs=\n" + if err := os.WriteFile(statePath, []byte(liveState), 0o600); err != nil { + t.Fatal(err) + } + marker := statePath + ".skip.abc123" + if err := os.WriteFile(marker, []byte("skipped=another run of this job is still in progress\noperation=op-7\ninputs=\n"), 0o600); err != nil { + t.Fatal(err) + } + + record, _, _ := runNotifierIn(t, base, app.ScheduledJob{Name: "nightly", Notify: []string{"failure", "timeout"}}, nil, map[string]string{ + "SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "abc123", + }) + if record["outcome"] != "skipped" || record["reason"] != "another run of this job is still in progress" { + t.Fatalf("a stand-aside activation was not recorded as skipped: %#v", record) + } + if record["operation"] != "op-7" { + t.Fatalf("the note's operation was lost: %#v", record) + } + body, err := os.ReadFile(statePath) + if err != nil || string(body) != liveState { + t.Fatalf("the running job's state was disturbed: %v %q", err, string(body)) + } + if _, err := os.Stat(marker); err == nil { + t.Fatal("the stand-aside note survived its own notifier") + } +} + +// The runner writes the stand-aside note and the notifier reads it. They name +// it with the same expression or the note is invisible, and the notifier goes +// back to the state file belonging to the run that is still going. +func TestScheduleSkipMarkerIsNamedIdenticallyOnBothSides(t *testing.T) { + names := app.Names{App: "sample", BasePath: "/var/lib/ob"} + job := app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1} + runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) + cfg := testConfig() + e := New(cfg, testProject(t), &transport.Fake{TargetName: "root@example.internal"}, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + notifier, err := e.scheduleNotifier(job) + if err != nil { + t.Fatal(err) + } + marker := regexp.MustCompile(`skip_marker="[^"]+"`) + inRunner := marker.FindString(runner) + inNotifier := marker.FindString(notifier) + if inRunner == "" || inRunner != inNotifier { + t.Fatalf("runner names the note %q and the notifier %q", inRunner, inNotifier) + } +} diff --git a/internal/engine/status.go b/internal/engine/status.go index def1fe48..386f9beb 100644 --- a/internal/engine/status.go +++ b/internal/engine/status.go @@ -171,20 +171,33 @@ func (e *Engine) Status(ctx context.Context) error { e.ui.Println(fmt.Sprintf("schedule %-11s %s", schedule.Name, e.ui.Warn(strings.Join(schedule.Issues, "; ")+" ⚠"))) continue } - result := schedule.LastResult - if result == "" { - result = "not run yet" - } + result := "not recorded" if schedule.Running { detail := fmt.Sprintf("running; policy: %s; timeout: %s", schedule.DeployLock, schedule.Timeout) + if schedule.Attempt > 0 { + detail += fmt.Sprintf("; attempt: %d", schedule.Attempt) + } if schedule.PinnedRelease != "" { detail += fmt.Sprintf("; release: %s; started: %s", schedule.PinnedRelease, schedule.StartedAt) } fmt.Fprintf(e.Opts.Out, "schedule %-11s %s\n", schedule.Name, detail) continue } - fmt.Fprintf(e.Opts.Out, "schedule %-11s active; policy: %s; timeout: %s; last: %s\n", - schedule.Name, schedule.DeployLock, schedule.Timeout, result) + detail := fmt.Sprintf("active; policy: %s; timeout: %s", schedule.DeployLock, schedule.Timeout) + if schedule.NextRun != "" { + detail += "; next: " + schedule.NextRun + } + switch { + case schedule.LastOutcome == "skipped": + result = "skipped (" + schedule.LastReason + ")" + case schedule.LastOutcome != "": + result = fmt.Sprintf("%s (%ds, %d attempt(s))", schedule.LastOutcome, schedule.LastDurationSeconds, schedule.LastAttempts) + } + detail += "; last: " + result + if !schedule.JournalPersistent { + detail += "; journal: volatile, history since boot only" + } + fmt.Fprintf(e.Opts.Out, "schedule %-11s %s\n", schedule.Name, detail) } if managed { diff --git a/internal/engine/status_snapshot_test.go b/internal/engine/status_snapshot_test.go index 0d3fe1c9..f8e01e67 100644 --- a/internal/engine/status_snapshot_test.go +++ b/internal/engine/status_snapshot_test.go @@ -248,6 +248,9 @@ ExecMainStatus=9 @@nightly:timer LoadState=loaded ActiveState=active +@@nightly:run +@@nightly:history +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:09Z","duration_s":8,"attempts":1,"exit_status":9,"outcome":"failure","inputs":{}} `}, true } return base(cmd) @@ -260,7 +263,7 @@ ActiveState=active if !snapshot.Complete || !snapshot.Diverged || len(snapshot.Schedules) != 1 { t.Fatalf("scheduled failure was not included as observed divergence: %#v", snapshot) } - if got := snapshot.Schedules[0]; !got.Diverged || got.LastResult != "exit-code" || got.LastExitStatus != 9 { + if got := snapshot.Schedules[0]; !got.Diverged || got.LastOutcome != "failure" { t.Fatalf("unexpected scheduled-job status: %#v", got) } } diff --git a/internal/notify/notify.go b/internal/notify/notify.go index 1dff3f07..86157fae 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -34,7 +34,13 @@ type Payload struct { Error string `json:"error,omitempty"` Operator string `json:"operator,omitempty"` TS string `json:"ts"` - Text string `json:"text"` // human line, filled by Send + // Skipped marks work that did not run rather than work that failed. It + // routes with the failures, because that is where an operator watches, + // but it must not be announced as one: a scheduled job that stood aside + // for a deploy did nothing wrong, and saying otherwise trains people to + // ignore the channel. + Skipped bool `json:"skipped,omitempty"` + Text string `json:"text"` // human line, filled by Send } // Request is the stable HTTP representation of one selected notification. @@ -55,14 +61,20 @@ func (p Payload) event() string { } func (p Payload) text() string { + id := p.DeployID + if id != "" { + id = " " + id + } if p.Status == "ok" { - id := p.DeployID - if id != "" { - id = " " + id - } return fmt.Sprintf("✅ %s: %s%s succeeded on %s", p.App, p.Verb, id, p.Host) } - return fmt.Sprintf("🚨 %s: %s FAILED on %s — %s", p.App, p.Verb, p.Host, p.Error) + if p.Skipped { + return fmt.Sprintf("⏭️ %s: %s%s did not run on %s — %s", p.App, p.Verb, id, p.Host, p.Error) + } + // The id belongs on the failure line most of all: a text webhook sends + // this sentence and nothing else, so without it the one notification an + // operator acts on cannot name the run they should go and read. + return fmt.Sprintf("🚨 %s: %s%s FAILED on %s — %s", p.App, p.Verb, id, p.Host, p.Error) } // Prepare renders a selected payload without sending it. An unset webhook and @@ -89,6 +101,11 @@ func Prepare(cfg app.Notification, p Payload) (*Request, error) { // outcome; operators use the trusted local diagnostics for details. if p.Status != "ok" && p.Error != "" { p.Error = "operation failed; inspect trusted local diagnostics" + if p.Skipped { + // The line already says the run did not happen, so this is only + // where to look for why. + p.Error = "inspect trusted host diagnostics" + } } p.Text = p.text() contentType := "application/json" diff --git a/internal/onebox/binding.go b/internal/onebox/binding.go index 66a82765..d40c1f67 100644 --- a/internal/onebox/binding.go +++ b/internal/onebox/binding.go @@ -31,7 +31,7 @@ func (s *Service) ResolveExecutionBinding(ctx context.Context, kind OperationKin func operationUsesInspectionRuntime(kind OperationKind) bool { switch kind { case KindResume, KindAbort, KindRollback, KindBootstrap, KindServiceApply, - KindProxyApply, KindScheduleApply, KindSecretsPush, KindDestroy, + KindProxyApply, KindScheduleApply, KindScheduleRun, KindSecretsPush, KindDestroy, // Backup operates on a service's data, never on the application's // release images, so a placeholder image must not stop a backup. KindBackupEnable, KindBackupDisable, KindBackupCreate, KindBackupPrune, KindAssuranceCheck, diff --git a/internal/onebox/execute.go b/internal/onebox/execute.go index 9928da21..d40864e1 100644 --- a/internal/onebox/execute.go +++ b/internal/onebox/execute.go @@ -208,6 +208,11 @@ func (s *Service) Execute(ctx context.Context, request ExecuteRequest) (Operatio case KindScheduleApply: result.EvidenceID = operationID err = e.ScheduleApply(ctx, operationID) + case KindScheduleRun: + result.EvidenceID = operationID + var run engine.ScheduleRunResult + run, err = e.ScheduleRun(ctx, operationID, request.Job, request.Inputs, request.Wait) + result.ScheduleRun = &run case KindSecretsPush: entries := encryptedEntries(lp.resolved) externalProjections := externalConnectionProjections(lp.resolved) diff --git a/internal/onebox/execution_types.go b/internal/onebox/execution_types.go index 7db71488..deb2c350 100644 --- a/internal/onebox/execution_types.go +++ b/internal/onebox/execution_types.go @@ -294,6 +294,12 @@ type ExecuteRequest struct { MigrationBackupOverride *MigrationBackupOverride BreakLock bool AllowDestructiveMounts bool + // Job, Inputs and Wait are the schedule_run arguments: a declared + // scheduled job, validated input overrides, and whether to block until + // the unit exits and return its run record. + Job string + Inputs map[string]string + Wait bool // Service is the backup operations' one argument. It is an input to a // mutation rather than a plan, because a backup stages nothing into a // release and has nothing to roll back. @@ -351,6 +357,9 @@ func (request ExecuteRequest) Validate() error { if (request.Approval != nil || request.BackupReport != nil || request.MigrationBackupOverride != nil) && request.Kind != KindDeploy && request.Kind != KindJobRun { return errors.New("approval and migration backup authorization are valid only for deploy and job run") } + if (request.Job != "" || len(request.Inputs) > 0 || request.Wait) && request.Kind != KindScheduleRun { + return errors.New("job, inputs and wait are valid only for schedule run") + } return nil } @@ -370,6 +379,7 @@ type OperationResult struct { MigrationBackupOverrideDigest string `json:"migration_backup_override_digest,omitempty"` Runner buildinfo.Runner `json:"runner"` JobResult *journal.JobResultEvidence `json:"job_result,omitempty"` + ScheduleRun *engine.ScheduleRunResult `json:"schedule_run,omitempty"` } func validateRunnerProvenance(runner buildinfo.Runner, planSchema string) error { diff --git a/internal/onebox/operation_errors.go b/internal/onebox/operation_errors.go index b5f3c32b..deda70a5 100644 --- a/internal/onebox/operation_errors.go +++ b/internal/onebox/operation_errors.go @@ -121,6 +121,22 @@ var operationFailureDefinitions = map[string]OperationFailure{ Message: "a release manifest is not valid closed JSON for its schema", Command: "ob status --output json", }, + "schedule_history_failed": { + Message: "the scheduled job's run records could not be read from the host journal", + Command: "ob status --output json", + }, + "schedule_input_invalid": { + Message: "an --input flag is not NAME=VALUE, or names the same input twice", + Command: "ob canonical --output json", + }, + "schedule_list_failed": { + Message: "the scheduled jobs' timer state could not be read", + Command: "ob status --output json", + }, + "schedule_logs_failed": { + Message: "the scheduled run's journal could not be read", + Command: "ob schedule history --output json", + }, "manifest_missing": { Message: "a release directory carries no manifest, so its lifecycle state is unknown", Command: "ob status --output json", diff --git a/internal/onebox/operation_types.go b/internal/onebox/operation_types.go index 101ca696..26e6e6f3 100644 --- a/internal/onebox/operation_types.go +++ b/internal/onebox/operation_types.go @@ -41,6 +41,7 @@ const ( KindSecretsPush OperationKind = "secrets_push" KindDestroy OperationKind = "destroy" KindJobRun OperationKind = "job_run" + KindScheduleRun OperationKind = "schedule_run" KindServiceImagePatch OperationKind = "service_image_patch" KindBackupEnable OperationKind = "backup_enable" @@ -477,7 +478,7 @@ func requireJSONEOF(decoder *json.Decoder) error { func validOperationKind(kind OperationKind) bool { switch kind { case KindDeploy, KindResume, KindAbort, KindRollback, KindBootstrap, KindJobRun, - KindServiceApply, KindProxyApply, KindScheduleApply, KindSecretsPush, KindDestroy, + KindServiceApply, KindProxyApply, KindScheduleApply, KindScheduleRun, KindSecretsPush, KindDestroy, KindServiceImagePatch, KindBackupEnable, KindBackupDisable, KindBackupCreate, KindBackupPrune, KindReplayArchive, KindRestoreTest, KindRestorePrepare, KindRestoreCutover, KindRestoreAbort, diff --git a/site/public/onebox.run-v1.schema.json b/site/public/onebox.run-v1.schema.json index ef540909..6c76f12e 100644 --- a/site/public/onebox.run-v1.schema.json +++ b/site/public/onebox.run-v1.schema.json @@ -731,7 +731,9 @@ "backup_key_material": { "description": "Key-material identities the backup report must name.", "examples": [ - "BACKUP_ACCESS_KEY_ID" + [ + "BACKUP_ACCESS_KEY_ID" + ] ], "items": { "type": "string" @@ -1123,7 +1125,10 @@ "type": "string" }, "on": { - "default": "success, failure", + "default": [ + "success", + "failure" + ], "description": "Operation outcomes that trigger this notification.", "items": { "enum": [ @@ -1878,6 +1883,11 @@ "required": [ "schedule" ] + }, + { + "required": [ + "inputs" + ] } ] } @@ -2231,6 +2241,64 @@ "description": "Run a minimal init process as PID 1 inside the container.", "type": "boolean" }, + "inputs": { + "additionalProperties": { + "additionalProperties": false, + "oneOf": [ + { + "required": [ + "enum" + ] + }, + { + "required": [ + "pattern" + ] + } + ], + "patternProperties": { + "^x-": {} + }, + "properties": { + "default": { + "description": "Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint.", + "type": "string" + }, + "description": { + "description": "What the input controls.", + "type": "string" + }, + "enum": { + "description": "Accepted values.", + "examples": [ + [ + "catalog" + ] + ], + "items": { + "type": "string" + }, + "type": "array" + }, + "pattern": { + "description": "Regular expression the whole value must match.", + "examples": [ + "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + ], + "type": "string" + } + }, + "required": [ + "default" + ], + "type": "object" + }, + "description": "Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them.", + "propertyNames": { + "pattern": "^[A-Z][A-Z0-9_]*$" + }, + "type": "object" + }, "labels": { "additionalProperties": {}, "description": "Additional container labels outside namespaces reserved by Onebox and the proxy.", @@ -2536,6 +2604,61 @@ ], "type": "string" }, + "notify": { + "default": [ + "failure", + "timeout" + ], + "description": "Run outcomes that send the configured notifications: success, failure, timeout, skipped.", + "items": { + "enum": [ + "success", + "failure", + "timeout", + "skipped" + ], + "type": "string" + }, + "type": "array" + }, + "retry": { + "additionalProperties": false, + "description": "Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run.", + "patternProperties": { + "^x-": {} + }, + "properties": { + "attempts": { + "default": 1, + "description": "Total attempts including the first, 1 to 10.", + "examples": [ + 3 + ], + "maximum": 10, + "minimum": 1, + "type": "integer" + }, + "backoff": { + "default": "30s", + "description": "Sleep before the second attempt; it doubles after each failure. Expects a duration such as 30s, 5m, 1h30m or 14d.", + "examples": [ + "1m" + ], + "pattern": "^(([0-9]+([.][0-9]+)?(ns|us|µs|ms|s|m|h))+|[0-9]+d)$", + "type": "string" + }, + "max_backoff": { + "default": "10m", + "description": "Upper bound for the doubling sleep. Expects a duration such as 30s, 5m, 1h30m or 14d.", + "examples": [ + "30m" + ], + "pattern": "^(([0-9]+([.][0-9]+)?(ns|us|µs|ms|s|m|h))+|[0-9]+d)$", + "type": "string" + } + }, + "type": "object" + }, "timeout": { "default": "1h", "description": "Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d.", diff --git a/site/src/content/docs/guides/schedule-a-job.mdx b/site/src/content/docs/guides/schedule-a-job.mdx index 527ee7f7..79087c48 100644 --- a/site/src/content/docs/guides/schedule-a-job.mdx +++ b/site/src/content/docs/guides/schedule-a-job.mdx @@ -1,11 +1,13 @@ --- title: Schedule a job -description: Bounded host timers, safe deployment coordination, failure status, and the cron forms Onebox refuses. -summary: How to declare a scheduled job, bound its run time, choose deployment and catch-up behavior, inspect failures, and understand which cron expressions are refused at load. +description: Bounded host timers, retry inside one firing, a record for every run, deployment coordination, manual runs with declared inputs, and the cron forms Onebox refuses. +summary: How to declare a scheduled job, bound its run time and retries, choose deployment and catch-up behavior, read run records back, run one now with inputs, and understand which cron expressions are refused at load. sidebar: order: 4 read_when: - "Adding a nightly or recurring task" + - "Working out why a scheduled run failed, or whether it ran at all" + - "Running a scheduled job now with different parameters" - "Understanding why a cron expression was refused" - "Working out how a scheduled job resolves secrets with no Onebox process running" --- @@ -42,15 +44,43 @@ result. Set a longer duration for jobs that legitimately need it. `catch_up` defaults to `true`: if the host was off at the scheduled time, the timer runs once after it returns. Set it to `false` for time-sensitive work that -should be skipped rather than run late. A failed run is not retried implicitly; -the next attempt is the next cron elapse. +should be skipped rather than run late. + +## Retry inside one firing + +By default a failed run is not retried; the next attempt is the next cron +elapse, which is the right answer for anything that fires every few minutes. +For a job whose failures are usually transient, a rate limit or an upstream +that was busy for a moment, declare a bounded retry: + +```yaml +schedule: + cron: "0 * * * *" + timeout: 45m + retry: {attempts: 3, backoff: 30s, max_backoff: 10m} +``` + +`attempts` counts every attempt including the first, so the default of `1` is +today's single run. After a non-zero exit the runner sleeps `backoff`, doubles +it after each further failure, and never sleeps longer than `max_backoff`. Every +attempt runs inside the same timer firing, under the same locks and the same +`timeout`; when the timeout expires the run ends, with no further attempt. + +Validation refuses a retry whose worst-case backoff is not smaller than the +timeout, because the last attempt could never start and the record would say +it did. With `deploy_lock: exclusive` the deploy lock is held through the +sleeps, so a deploy waits for the run to finish; with `deploy_lock: pinned` the +release lease is held instead and every attempt runs the same release. ## Deployment coordination is exclusive by default A host-fired job takes an application-wide kernel lock for its whole container run. Every operation that establishes Onebox's fenced application lock takes the same scheduling mutex while publishing its ownership. A timer that collides -fails visibly instead of modifying Docker beside a deploy. +does not modify Docker beside a deploy: it records a `skipped` run with the +reason and exits cleanly. An application lock older than its TTL is treated as +expired, exactly as a deploy treats it, so a runner that died mid-operation +cannot silence a timer forever. The target must provide `flock` (part of `util-linux` on supported Linux hosts). Onebox refuses to install or run schedules when that serialization primitive is @@ -102,23 +132,77 @@ under the same application lock, schedule mutex, fence, and journal boundary as a deploy, but it does not stage or activate a release. Run it after upgrading Onebox when an application may not be deployed again soon. -## Failures remain visible +## Every run leaves a record -systemd retains the last oneshot result. `ob status` reads it alongside the -timer state; a non-success result or inactive timer is reported as divergence: +When a run ends, for any reason, the unit's `ExecStopPost` writes one record +to the host journal with the syslog identifier `ob-run` and the job's unit in +its `ONEBOX_UNIT` field, so `journalctl SYSLOG_IDENTIFIER=ob-run +ONEBOX_UNIT=ob-- -o cat` on the host is the raw history: + +```json +{"run":"a3f9…","job":"nightly-dump","trigger":"timer","operation":"","release":"20260905-140000-ab12cd","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:04:37Z","duration_s":276,"attempts":2,"exit_status":0,"outcome":"success","reason":"","inputs":{}} +``` + +`run` is systemd's invocation id, so the record and the run's own log share a +key. `outcome` is one of `success`, `failure`, `timeout`, or `skipped`. A skip +is a firing that met a running instance of the same job or an application +operation holding the deploy lock. The runner writes the `reason` and exits +before any container starts, so the unit is not failed and the job's own exit +status is never mistaken for a skip. Skips are recorded because a job that is +silently never running looks exactly like one that works: one skip is timing, +and three in a row are reported by `ob status` as an issue. + +A skip is news about timing, not about the job, so it never clears a failure: +`ob status` keeps reporting the newest run that actually happened, and says +that nothing has run since. A job that selects `skipped` in `notify` is told +its run did not happen, not that it failed. + +Read the records back from the workstation: + +```sh +ob schedule list # every job, its timer state and next elapse +ob schedule history nightly-dump # records, newest first; -n 50 for more +ob schedule logs nightly-dump # the journal of the newest run +ob schedule logs nightly-dump --run a3f9… +``` + +`ob status` reads the same records. Each scheduled job's line carries the next +elapse, the last outcome with its duration and attempt count, and, while a run +is in progress, the attempt it is on. A last outcome of `failure` or `timeout` +is reported as divergence: ```text -schedule nightly-dump last run failed: timeout (exit 15) ⚠ +schedule nightly-dump last run failed: timeout (exit 143) ⚠ +``` + +Retention is the journal's. A host whose journal lives in memory keeps records +only since its last boot; `ob status` says so on the job's line, and the fix is +a persistent journal (`/var/log/journal`), which the supported images have by +default. + +### Notifications per outcome + +`schedule.notify` selects which outcomes send the configured notifications. The +default is `[failure, timeout]`, which is what the failure notifier always did. +Add `success` for a job whose completion someone waits for, and `skipped` for a +job whose firings collide often enough that silence would hide it: + +```yaml +schedule: + cron: "0 2 * * *" + notify: [failure, timeout, skipped] ``` -Use `journalctl -u ob--.service` on the target for trusted diagnostic -output. Every configured notification selecting `failure` also receives a -bounded, fail-open POST when a host-fired run fails or times out. Onebox writes -the webhook handler root-only beside the unit, so credentials in webhook -paths do not appear in `ExecStart` or `ExecStopPost`. A delivery failure is -recorded in the unit's journal and never replaces the job's original result. -The webhook must be reachable from the target host; timer delivery uses its -network path, not the operator workstation's. +Each selected outcome sends a bounded, fail-open POST to every webhook whose +own `on` list accepts that class of outcome. The payload carries the run id as +its `deploy_id`, so `ob schedule logs --run ` finds the run; it +carries nothing else about the run, because notifications cross the host trust +boundary and diagnostics stay on the host. Onebox writes the webhook handler +root-only beside the unit, so credentials in webhook paths do not appear in +`ExecStart` or `ExecStopPost`. A delivery failure is recorded in the unit's +journal and never replaces the job's own result. The webhook must be reachable +from the target host; timer delivery uses its network path, not the operator +workstation's. ## Cron is translated exactly, or refused @@ -140,6 +224,73 @@ load rather than running on days nobody chose. `none`; a nightly prune is `destructive`. The rollback and abort gates read it, and a job that lies about it defeats them. +## Run one now with inputs + +A scheduled job may declare inputs: named parameters that reach the container +as environment variables. A timer firing uses the defaults; an operator may run +the job now and override them. + +```yaml +workloads: + source-sync: + role: job + image: ghcr.io/acme/ingest:1.8.2 + command: ["./ingest", "sync"] + data_effect: none + inputs: + SOURCE: + enum: [catalog, prices, reviews] + default: catalog + description: Which upstream to sync. + SINCE: + pattern: '^([0-9]{4}-[0-9]{2}-[0-9]{2})?$' + default: "" + description: Only records changed since this date. Empty means the stored cursor. + schedule: + cron: "0 * * * *" + deploy_lock: pinned +``` + +```sh +ob schedule run source-sync --input SOURCE=prices --input SINCE=2026-09-01 --wait +``` + +Names are upper-case identifiers outside Onebox's `ONEBOX_` namespace and may +not repeat an `env` key. Each input declares exactly one of `enum` or +`pattern`, and a `default` that satisfies it; a pattern matches the whole +value. Whatever the pattern allows, a value may not contain a double quote, a +backslash, or a control character, and is at most 256 bytes. Those rules are +what let the runner hand values to the container without escaping anything. + +`ob schedule run` validates every value on the workstation, refuses if the +unit is already running or a manual run is still pending, journals the request +as `schedule_run` with the operator and the inputs, and starts the unit. It +holds the application lock only while writing and journaling, because the +runner skips a run that meets that lock. The host record of the run carries the +operation id, so `ob audit` and `ob schedule history` join on it. Without +`--wait` the command returns as soon as the unit is started; with it, the +command blocks until the unit exits and reports the record. + +Only a job with `data_effect: none` accepts inputs or manual runs this way. A +`migration` or `destructive` job is still run by its timer as declared, but an +operator-initiated run of it keeps the sealed plan, approval and backup-report +gates of `ob job run` below. + +The runner tells a manual activation from a timer firing by the `TRIGGER_UNIT` +variable systemd sets on timer activations, which that project introduced in +version 252. Ubuntu 24.04 and Debian 12 qualify; Ubuntu 22.04 and Debian 11 do +not. + +Scheduled jobs still run on those older hosts, unchanged. Two things narrow: + +- A job that declares `inputs` is refused, by `ob preflight` and by `ob deploy`, + before anything is staged. Without `TRIGGER_UNIT` the next timer firing would + read the file meant for an operator's run. +- `ob schedule run` is refused for the same reason. The timer keeps firing. + +The records on such a host say `trigger: unknown`, because the runner cannot +observe what started it and will not guess. + ## Running one by hand ```sh diff --git a/site/src/content/docs/reference/cli.mdx b/site/src/content/docs/reference/cli.mdx index 08c6935e..349f577a 100644 --- a/site/src/content/docs/reference/cli.mdx +++ b/site/src/content/docs/reference/cli.mdx @@ -929,7 +929,8 @@ Manage the systemd timers generated for scheduled jobs. Timers outlive the Onebox process and the package installed on the operator workstation. `apply` explicitly reconciles their units after a runner or -configuration change without deploying a release. +configuration change without deploying a release. `list`, `history` and `logs` +read the timer state and the run records the host keeps in its journal. Usage: ob schedule [flags] @@ -937,6 +938,10 @@ Usage: Available Commands: apply reconcile scheduled-job units without deploying a release + history run records of one scheduled job, newest first + list declared scheduled jobs with timer state and next elapse + logs journal of one scheduled run + run start a scheduled job now with declared inputs Flags: -h, --help help for schedule @@ -971,6 +976,92 @@ Global Flags: -v, --verbose print every remote command ``` +### ob schedule history + +``` +Read the run records the host wrote for one scheduled job. Each record is one activation: run id, trigger, release, start and end, attempts, exit status, outcome and, for a manual run, its inputs. + +Records live in the host journal with syslog identifier ob-run and the job's unit in their ONEBOX_UNIT field; retention is the journal's. Reads only. + +Usage: + ob schedule history [flags] + +Flags: + -n, --count int number of newest runs to show (default 20) + -h, --help help for history + +Global Flags: + -c, --config string path to the project YAML file (default "ob.yml") + -e, --env string environment name (default "production") + --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") + -v, --verbose print every remote command +``` + +### ob schedule list + +``` +List every job that declares a schedule beside what the host's timer says: whether it is active, when it fires next, and when it last fired. Reads only. + +Usage: + ob schedule list [flags] + +Flags: + -h, --help help for list + +Global Flags: + -c, --config string path to the project YAML file (default "ob.yml") + -e, --env string environment name (default "production") + --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") + -v, --verbose print every remote command +``` + +### ob schedule logs + +``` +Stream the host journal for one run of a scheduled job: by default the newest recorded run, or the run named with --run. The run id is systemd's invocation id, so the output is exactly that activation. Reads only. + +Log bytes are operator-controlled and may contain secrets; Onebox does not claim +to redact passthrough output. + +Usage: + ob schedule logs [flags] + +Flags: + -h, --help help for logs + --run string run id from ob schedule history; default the newest run + +Global Flags: + -c, --config string path to the project YAML file (default "ob.yml") + -e, --env string environment name (default "production") + --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") + -v, --verbose print every remote command +``` + +### ob schedule run + +``` +Start one scheduled job's unit now, with values for its declared inputs. Values are validated on the workstation against the declaration; an undeclared name or a value outside its enum or pattern is refused before anything reaches the host. + +Only a job with data_effect none may run this way; a migration or destructive job keeps the sealed plan of ob job run. The request is journaled as schedule_run with the operator and inputs, and the host record carries the operation id, so ob audit and ob schedule history join on it. + +The outcome is the run record: ob schedule history , or --wait to block until the unit exits and print it. + +Usage: + ob schedule run [flags] + +Flags: + --break-lock break a stale operation lock after inspecting its holder + -h, --help help for run + --input stringArray input override as NAME=VALUE; repeatable + --wait block until the unit exits and report the run record + +Global Flags: + -c, --config string path to the project YAML file (default "ob.yml") + -e, --env string environment name (default "production") + --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") + -v, --verbose print every remote command +``` + ## ob schema ``` diff --git a/site/src/content/docs/reference/errors.mdx b/site/src/content/docs/reference/errors.mdx index 99fe77b8..acd1493e 100644 --- a/site/src/content/docs/reference/errors.mdx +++ b/site/src/content/docs/reference/errors.mdx @@ -152,6 +152,10 @@ step to complete rather than a line to run verbatim. | `preflight_failed` | a target readiness check failed before any mutation | — | — | | `recovery_incomplete` | recovery did not reach its verified terminal state | resolving | `ob resume --output ndjson` | | `rollback_target_missing` | no previously serving release is recorded as a rollback target | next | `ob plan --output json` | +| `schedule_history_failed` | the scheduled job's run records could not be read from the host journal | diagnostic | `ob status --output json` | +| `schedule_input_invalid` | an --input flag is not NAME=VALUE, or names the same input twice | diagnostic | `ob canonical --output json` | +| `schedule_list_failed` | the scheduled jobs' timer state could not be read | diagnostic | `ob status --output json` | +| `schedule_logs_failed` | the scheduled run's journal could not be read | next | `ob schedule history --output json` | | `secret_cleanup_pending` | the rotation is applied and verified, but removing the retired generation did not finish | resolving | `ob secrets push --output ndjson` | | `secret_declaration_not_deployed` | the deployed release does not declare this secret graph | next | `ob plan --output json` | | `secret_entry_not_selected` | more than one editable secret source exists, so an entry identifier is required | diagnostic | `ob secrets list --output json` | diff --git a/site/src/content/docs/reference/fields/workloads.mdx b/site/src/content/docs/reference/fields/workloads.mdx index aa9bcc62..cfd038ee 100644 --- a/site/src/content/docs/reference/fields/workloads.mdx +++ b/site/src/content/docs/reference/fields/workloads.mdx @@ -19,7 +19,7 @@ cannot drift from what `ob validate` accepts. ## Fields on this page -`args` · `bind` · `build` · `catch_up` · `command` · `compose` · `condition` · `container` · `context` · `cpus` · `cron` · `data_effect` · `deploy_lock` · `dockerfile` · `domain` · `drain` · `driver` · `entrypoint` · `env` · `env_files` · `exec` · `extra_hosts` · `file` · `grace` · `health` · `host` · `hostname` · `http` · `image` · `init` · `interval` · `labels` · `logging` · `memory` · `middlewares` · `mode` · `name` · `needs` · `options` · `path` · `persistence` · `port` · `protocol` · `provider` · `published_ports` · `pull` · `reference` · `replicas` · `resources` · `retries` · `role` · `routes` · `schedule` · `scheme` · `signal` · `source` · `start_period` · `stdin_open` · `strategy` · `target` · `tcp` · `timeout` · `timezone` · `tls` · `tty` · `user` · `volumes` · `wait` · `when` · `within` · `working_dir` +`args` · `attempts` · `backoff` · `bind` · `build` · `catch_up` · `command` · `compose` · `condition` · `container` · `context` · `cpus` · `cron` · `data_effect` · `default` · `deploy_lock` · `description` · `dockerfile` · `domain` · `drain` · `driver` · `entrypoint` · `enum` · `env` · `env_files` · `exec` · `extra_hosts` · `file` · `grace` · `health` · `host` · `hostname` · `http` · `image` · `init` · `inputs` · `interval` · `labels` · `logging` · `max_backoff` · `memory` · `middlewares` · `mode` · `name` · `needs` · `notify` · `options` · `path` · `pattern` · `persistence` · `port` · `protocol` · `provider` · `published_ports` · `pull` · `reference` · `replicas` · `resources` · `retries` · `retry` · `role` · `routes` · `schedule` · `scheme` · `signal` · `source` · `start_period` · `stdin_open` · `strategy` · `target` · `tcp` · `timeout` · `timezone` · `tls` · `tty` · `user` · `volumes` · `wait` · `when` · `within` · `working_dir` ## Reference @@ -58,6 +58,11 @@ cannot drift from what `ob validate` accepts. | `.image.pull` | `always` · `missing` · `never` | `missing` | When to fetch the image from the registry: missing fetches only what the host does not already hold, always fetches every release, never fetches at all and fails on a missing image. | | `.image.reference` | string | — | Complete container image reference, optionally tagged or digest-pinned. Expects a registry reference such as nginx:1.27 or ghcr.io/acme/app@sha256:…. | | `.init` | boolean | — | Run a minimal init process as PID 1 inside the container. | +| `.inputs` | map | — | Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them. | +| `.inputs..default` `*` | string | — | Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint. | +| `.inputs..description` | string | — | What the input controls. | +| `.inputs..enum` | list | — | Accepted values. | +| `.inputs..pattern` | string | — | Regular expression the whole value must match. | | `.labels` | map | — | Additional container labels outside namespaces reserved by Onebox and the proxy. | | `.logging` | object | — | Container logging driver and driver-specific options. | | `.logging.driver` | string | — | Container runtime logging driver. Expects a log driver name such as local, json-file or an org/plugin:tag. | @@ -92,6 +97,11 @@ cannot drift from what `ob validate` accepts. | `.schedule.catch_up` | boolean | `true` | Run once after the host returns if an elapsed schedule was missed while it was offline. | | `.schedule.cron` | string | — | Five-field cron schedule translated to a host timer. Expects five cron fields. | | `.schedule.deploy_lock` | `exclusive` · `pinned` | `exclusive` | Deployment coordination policy: exclusive blocks application operations for the full run; pinned leases the immutable starting release and permits only deployments without data-changing jobs or untyped hooks. | +| `.schedule.notify` | list of `success` · `failure` · `timeout` · `skipped` | `failure, timeout` | Run outcomes that send the configured notifications: success, failure, timeout, skipped. | +| `.schedule.retry` | object | — | Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run. | +| `.schedule.retry.attempts` | integer | `1` | Total attempts including the first, 1 to 10. | +| `.schedule.retry.backoff` | string | `30s` | Sleep before the second attempt; it doubles after each failure. Expects a duration such as 30s, 5m, 1h30m or 14d. | +| `.schedule.retry.max_backoff` | string | `10m` | Upper bound for the doubling sleep. Expects a duration such as 30s, 5m, 1h30m or 14d. | | `.schedule.timeout` | string | `1h` | Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d. | | `.schedule.timezone` | string | `UTC` | IANA timezone used to interpret the cron schedule. Expects an IANA zone name such as UTC or Europe/Berlin. | | `.stdin_open` | boolean | — | Keep standard input open for the container. | diff --git a/site/src/content/docs/reference/policies.mdx b/site/src/content/docs/reference/policies.mdx index 16f07916..a1fd6505 100644 --- a/site/src/content/docs/reference/policies.mdx +++ b/site/src/content/docs/reference/policies.mdx @@ -96,9 +96,9 @@ redacted. | Class | JSON | NDJSON | Commands | | --- | --- | --- | --- | -| Finite envelope | yes | no | `ob approve` · `ob audit` · `ob backup status` · `ob canonical` · `ob doctor` · `ob eject` · `ob init` · `ob job plan` · `ob plan` · `ob preflight` · `ob preview` · `ob schema` · `ob secrets list` · `ob status` · `ob validate` · `ob version` | -| Finite operation stream | yes | yes | `ob abort` · `ob backup create` · `ob backup enable` · `ob backup disable` · `ob backup drill` · `ob backup prune` · `ob backup restore` · `ob backup verify` · `ob bootstrap` · `ob deploy` · `ob destroy` · `ob job run` · `ob proxy apply` · `ob resume` · `ob rollback` · `ob schedule apply` · `ob secrets push` · `ob service apply` | -| Operator passthrough | finite only | yes | `ob logs` | +| Finite envelope | yes | no | `ob approve` · `ob audit` · `ob backup status` · `ob canonical` · `ob doctor` · `ob eject` · `ob init` · `ob job plan` · `ob plan` · `ob preflight` · `ob preview` · `ob schedule history` · `ob schedule list` · `ob schema` · `ob secrets list` · `ob status` · `ob validate` · `ob version` | +| Finite operation stream | yes | yes | `ob abort` · `ob backup create` · `ob backup enable` · `ob backup disable` · `ob backup drill` · `ob backup prune` · `ob backup restore` · `ob backup verify` · `ob bootstrap` · `ob deploy` · `ob destroy` · `ob job run` · `ob proxy apply` · `ob resume` · `ob rollback` · `ob schedule apply` · `ob schedule run` · `ob secrets push` · `ob service apply` | +| Operator passthrough | finite only | yes | `ob logs` · `ob schedule logs` | | Operator passthrough | no | yes | `ob exec` | | Trusted editor | yes, after exit | no | `ob secrets edit` | diff --git a/site/src/content/docs/status/capabilities.mdx b/site/src/content/docs/status/capabilities.mdx index 7c99867a..556a78e5 100644 --- a/site/src/content/docs/status/capabilities.mdx +++ b/site/src/content/docs/status/capabilities.mdx @@ -27,6 +27,11 @@ This page is the reconciliation. Three states: additive. - Compose generation and validation, SSH transport with known-host checking, state-bound plans, image pinning, rendered diffs. +- Scheduled jobs as host timers with exact cron translation, bounded retry + inside one firing, one run record per activation in the host journal + (`ob schedule history`, `ob schedule logs`, `ob status`), per-outcome + notifications, and declared inputs for operator-initiated runs + (`ob schedule run`). - Per-workload reconciliation that retains proven-unchanged healthy workloads, health-gated rolling or recreate replacement for changed or uncertain ones, traffic drain, verification, versioned releases, retention, rollback.