diff --git a/test/e2e/README.md b/test/e2e/README.md index 60a7b79d71..237fce7bbd 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -244,6 +244,32 @@ The `Makefile` automatically separates scenarios when run without additional `GO ## Running Tests +### Automatic PSA checks + +When `PSA_CHECK_BIN` points to the `cluster-debug-tools` `kubectl-dev_tool` +binary, the shared scenario cleanup hook runs `psa-check` for every scenario +before its namespace is deleted. No feature-file changes are required for new +scenarios. The check evaluates the scenario namespace against the `restricted` +profile and uses the same `KUBECONFIG` as the E2E test. + +Build the checker separately, then run the suite with checks enabled: + +```bash +PSA_CHECK_BIN=$HOME/bin/kubectl-dev_tool \ +PSA_CHECK_REQUIRED=true \ +ARTIFACT_PATH=/tmp/operator-controller-artifacts \ +make test-e2e +``` + +`PSA_CHECK_REQUIRED=true` makes an unavailable checker fail the scenario. A +configured checker always makes reported violations fail the scenario. Without +the required setting, an unavailable checker is logged and skipped, which keeps +local E2E runs usable on machines without the plugin. +When `ARTIFACT_PATH` is set, each scenario writes its JSON result under +`psa///psa.json`; stderr is written beside it when +present. The check also runs for scenarios that already failed, while the +existing cleanup behavior continues to preserve failed-scenario resources. + ### Run All Tests ```bash diff --git a/test/e2e/steps/hooks.go b/test/e2e/steps/hooks.go index ee93303408..7ab3dbe346 100644 --- a/test/e2e/steps/hooks.go +++ b/test/e2e/steps/hooks.go @@ -268,6 +268,13 @@ func ScenarioCleanup(ctx context.Context, _ *godog.Scenario, err error) (context } } + if psaErr := runPSACheck(ctx, sc); psaErr != nil { + if err != nil { + return ctx, errors.Join(err, psaErr) + } + return ctx, psaErr + } + if err != nil { return ctx, err } diff --git a/test/e2e/steps/psa.go b/test/e2e/steps/psa.go new file mode 100644 index 0000000000..552b1db64f --- /dev/null +++ b/test/e2e/steps/psa.go @@ -0,0 +1,129 @@ +package steps + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/spf13/pflag" +) + +var ( + psaCheckBin string + psaCheckRequired bool +) + +func init() { + flagSet := pflag.CommandLine + flagSet.StringVar(&psaCheckBin, "psa.check-bin", os.Getenv("PSA_CHECK_BIN"), "Path to the cluster-debug-tools PSA checker; empty disables automatic PSA checks") + flagSet.BoolVar(&psaCheckRequired, "psa.check-required", psaCheckRequiredFromEnv(), "Fail scenarios when the PSA checker is unavailable") +} + +func psaCheckRequiredFromEnv() bool { + required, err := strconv.ParseBool(os.Getenv("PSA_CHECK_REQUIRED")) + if err != nil { + return false + } + return required +} + +// runPSACheck evaluates the current scenario namespace before cleanup removes it. +// The checker is intentionally external because psa-check is distributed as a +// kubectl-dev_tool plugin rather than a package consumed by this repository. +func runPSACheck(ctx context.Context, sc *scenarioContext) error { + bin := strings.TrimSpace(psaCheckBin) + if bin == "" { + if psaCheckRequired { + return fmt.Errorf("PSA checker is required but --psa.check-bin or PSA_CHECK_BIN is unset") + } + return nil + } + + if _, err := exec.LookPath(bin); err != nil { + if psaCheckRequired { + return fmt.Errorf("PSA checker %q is unavailable: %w", bin, err) + } + logger.Info("Skipping PSA check because checker is unavailable", "binary", bin, "error", err) + return nil + } + + cmd := exec.CommandContext(ctx, bin, + "psa-check", + "--namespace", sc.namespace, + "--level", "restricted", + "--output", "json", + ) + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath)) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + + artifactPath, artifactErr := psaArtifactPath(sc) + if artifactErr != nil { + return artifactErr + } + if artifactPath != "" { + if writeErr := os.WriteFile(artifactPath, stdout.Bytes(), 0o600); writeErr != nil { + return fmt.Errorf("write PSA result for scenario %q: %w", sc.scenarioName, writeErr) + } + if stderr.Len() > 0 { + stderrPath := strings.TrimSuffix(artifactPath, filepath.Ext(artifactPath)) + ".stderr" + if writeErr := os.WriteFile(stderrPath, stderr.Bytes(), 0o600); writeErr != nil { + return fmt.Errorf("write PSA stderr for scenario %q: %w", sc.scenarioName, writeErr) + } + } + } + + if err == nil { + return nil + } + + message := strings.TrimSpace(stdout.String()) + if stderrMessage := strings.TrimSpace(stderr.String()); stderrMessage != "" { + if message != "" { + message += "; " + } + message += stderrMessage + } + if message == "" { + message = "no diagnostic output" + } + return fmt.Errorf("PSA check failed for scenario %q in namespace %q: %w: %s", sc.scenarioName, sc.namespace, err, message) +} + +func psaArtifactPath(sc *scenarioContext) (string, error) { + basePath := strings.TrimSpace(os.Getenv("ARTIFACT_PATH")) + if basePath == "" { + return "", nil + } + + path := filepath.Join(basePath, "psa", sanitizePSAArtifactPart(sc.featureName), sanitizePSAArtifactPart(sc.id)) + if err := os.MkdirAll(path, 0o755); err != nil { + return "", fmt.Errorf("create PSA artifact directory %q: %w", path, err) + } + return filepath.Join(path, "psa.json"), nil +} + +func sanitizePSAArtifactPart(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "unknown" + } + var b strings.Builder + for _, r := range value { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_', r == '.': + b.WriteRune(r) + default: + b.WriteByte('-') + } + } + return strings.Trim(b.String(), "-") +} diff --git a/test/e2e/steps/psa_test.go b/test/e2e/steps/psa_test.go new file mode 100644 index 0000000000..7122f24316 --- /dev/null +++ b/test/e2e/steps/psa_test.go @@ -0,0 +1,92 @@ +package steps + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunPSACheck(t *testing.T) { + tests := []struct { + name string + script string + required bool + wantErr bool + wantOutput string + }{ + { + name: "successful check writes JSON artifact", + script: "printf '%s' '{\"items\":[]}'", + wantOutput: `{"items":[]}`, + }, + { + name: "violations fail the check and preserve output", + script: "printf '%s' '{\"items\":[{\"namespace\":\"ns-test\"}]}' ; exit 1", + wantErr: true, + wantOutput: `{"items":[{"namespace":"ns-test"}]}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bin := filepath.Join(t.TempDir(), "psa-check") + if err := os.WriteFile(bin, []byte("#!/bin/sh\n"+tt.script+"\n"), 0o700); err != nil { + t.Fatal(err) + } + + artifactPath := t.TempDir() + t.Setenv("ARTIFACT_PATH", artifactPath) + oldBin, oldRequired, oldKubeconfig := psaCheckBin, psaCheckRequired, kubeconfigPath + psaCheckBin, psaCheckRequired, kubeconfigPath = bin, tt.required, "/tmp/test.kubeconfig" + t.Cleanup(func() { + psaCheckBin, psaCheckRequired, kubeconfigPath = oldBin, oldRequired, oldKubeconfig + }) + + sc := &scenarioContext{id: "scenario-1", featureName: "install", scenarioName: tt.name, namespace: "ns-test"} + err := runPSACheck(context.Background(), sc) + if (err != nil) != tt.wantErr { + t.Fatalf("runPSACheck() error = %v, want error: %t", err, tt.wantErr) + } + + result, err := os.ReadFile(filepath.Join(artifactPath, "psa", "install", "scenario-1", "psa.json")) + if err != nil { + t.Fatal(err) + } + if string(result) != tt.wantOutput { + t.Fatalf("PSA artifact = %q, want %q", result, tt.wantOutput) + } + }) + } +} + +func TestRunPSACheckConfiguration(t *testing.T) { + t.Run("optional checker can be disabled", func(t *testing.T) { + oldBin, oldRequired := psaCheckBin, psaCheckRequired + psaCheckBin, psaCheckRequired = "", false + t.Cleanup(func() { psaCheckBin, psaCheckRequired = oldBin, oldRequired }) + + err := runPSACheck(context.Background(), &scenarioContext{namespace: "ns-test"}) + if err != nil { + t.Fatalf("runPSACheck() error = %v", err) + } + }) + + t.Run("required checker must be configured", func(t *testing.T) { + oldBin, oldRequired := psaCheckBin, psaCheckRequired + psaCheckBin, psaCheckRequired = "", true + t.Cleanup(func() { psaCheckBin, psaCheckRequired = oldBin, oldRequired }) + + err := runPSACheck(context.Background(), &scenarioContext{namespace: "ns-test"}) + if err == nil || !strings.Contains(err.Error(), "checker is required") { + t.Fatalf("runPSACheck() error = %v, want required-checker error", err) + } + }) +} + +func TestSanitizePSAArtifactPart(t *testing.T) { + if got, want := sanitizePSAArtifactPart("feature/name with spaces"), "feature-name-with-spaces"; got != want { + t.Fatalf("sanitizePSAArtifactPart() = %q, want %q", got, want) + } +}