diff --git a/README.md b/README.md index 59f8dfe..e6fe62d 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,56 @@ once because another top-level flow calls it via `runFlow`. --- +### Status, artifacts and list + +Commands for working with Maestro projects after they were started, typically together with `--async`. Every command accepts `--api-key` / `--api-secret`, `--debug`, and the `--json`, `--json-file`, `--json-file-name` output flags described under [JSON Output](#json-output). + +```sh +# Start tests without waiting and capture the project id +testingbot maestro app.apk ./flows --async --json | jq -r .appId + +# Check on it later; --wait blocks with live progress and exits 2 on failure +testingbot status --id 1234 +testingbot status --id 1234 --wait + +# Fetch reports and artifacts once it finished +testingbot artifacts --id 1234 --report junit --report-output-dir ./reports +testingbot artifacts --id 1234 --download-artifacts failed --artifacts-output-dir ./artifacts + +# Browse recent projects +testingbot list +testingbot list --count 25 --offset 25 --json +``` + +**`status --id `** + +| Option | Description | +|--------|-------------| +| `-w, --wait` | Block until every run has finished, showing the same live flow table as a foreground run | +| `-q, --quiet` | Suppress progress output | + +Exit code is `0` while the project is still running (JSON `outcome: "running"`), `0`/`2` once it completed, `1` on errors. + +**`artifacts --id `** + +| Option | Description | +|--------|-------------| +| `--report ` | Download report: `html`, `html-detailed` or `junit` | +| `--report-output-dir ` | Directory to save reports (required with `--report`) | +| `--download-artifacts [mode]` | Download logs, screenshots and video. Mode: `all` (default) or `failed` | +| `--artifacts-output-dir ` | Directory to save the artifacts zip (defaults to current directory) | + +Fails with exit code `1` if the project is still running; use `status --wait` first. + +**`list`** + +| Option | Description | +|--------|-------------| +| `--count ` | Maximum number of projects to return (default 10) | +| `--offset ` | Number of projects to skip, for pagination | + +Projects are listed newest first with id, name, state, run and flow counts. `--json` returns `{ provider, meta: { offset, count, total }, projects: [...] }` with a dashboard `url` per project. + ### Espresso Run Android Espresso tests on real devices and emulators. diff --git a/src/cli.ts b/src/cli.ts index d3cb600..6c18037 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,4 +1,4 @@ -import { Command } from 'commander'; +import { Command, InvalidArgumentError } from 'commander'; import logger, { enableDebugLogging } from './logger'; import Auth from './auth'; import Espresso from './providers/espresso'; @@ -22,6 +22,7 @@ import MaestroOptions, { } from './models/maestro_options'; import Maestro from './providers/maestro'; import Login from './providers/login'; +import Credentials from './models/credentials'; import TestingBotError from './models/testingbot_error'; import { redirectLogsToStderr } from './logger'; import { @@ -56,6 +57,37 @@ function jsonOptionsFrom(args: JsonCliArgs): JsonOutputOptions { return options; } +/** Resolves credentials from flags, env or ~/.testingbot, or fails with guidance. */ +async function requireCredentials(args: { + apiKey?: string; + apiSecret?: string; +}): Promise { + const credentials = await Auth.getCredentials({ + apiKey: args.apiKey, + apiSecret: args.apiSecret, + }); + if (credentials === null) { + throw new TestingBotError( + 'No TestingBot credentials found. Please authenticate using one of these methods:\n' + + ' 1. Run "testingbot login" to authenticate via browser (recommended)\n' + + ' 2. Use --api-key and --api-secret options\n' + + ' 3. Set TB_KEY and TB_SECRET environment variables\n' + + ' 4. Create ~/.testingbot file with content: key:secret', + ); + } + return credentials; +} + +function parseProjectId(value: string): number { + const id = Number.parseInt(value, 10); + if (!Number.isInteger(id) || id <= 0) { + throw new InvalidArgumentError( + `expected a positive integer (the "Project ID" printed when a run starts), got "${value}".`, + ); + } + return id; +} + /** Emits JSON output (if requested) and sets the exit code for a finished run. */ async function finishCommand( output: JsonOutput, @@ -808,6 +840,245 @@ program }) .showHelpAfterError(true); +const JSON_FLAGS = [ + ['--json', 'Print results as JSON on stdout; logs move to stderr.'], + [ + '--json-file', + 'Write results as JSON to a file (default: _testingbot.json).', + ], + [ + '--json-file-name ', + 'Custom path for the JSON results file (requires --json-file).', + ], +] as const; + +const AUTH_FLAGS = [ + ['--api-key ', 'TestingBot API key.'], + ['--api-secret ', 'TestingBot API secret.'], + ['--debug', 'Enable debug logging of API responses.'], +] as const; + +function withFlags( + command: Command, + flags: ReadonlyArray, +): Command { + for (const [flag, description] of flags) { + command.option(flag, description); + } + return command; +} + +withFlags( + withFlags( + program + .command('status') + .description( + 'Show the current state of a Maestro project started earlier (e.g. with --async).', + ) + .requiredOption( + '--id ', + 'Project ID printed when the run started.', + parseProjectId, + ) + .option( + '-w, --wait', + 'Block until every run has finished, showing live progress. Exits 2 if any flow failed.', + ) + .option( + '-q, --quiet', + 'Quieter console output without progress updates.', + ), + JSON_FLAGS, + ), + AUTH_FLAGS, +) + .action(async (args) => { + let jsonOptions: JsonOutputOptions | undefined; + try { + jsonOptions = jsonOptionsFrom(args); + const credentials = await requireCredentials(args); + if (args.debug) enableDebugLogging(); + const maestro = new Maestro( + credentials, + MaestroOptions.forExistingProject({ + quiet: args.quiet || jsonOptions.json || jsonOptions.jsonFile, + debug: args.debug, + }), + ); + const result = await maestro.status(args.id, { wait: args.wait }); + await finishCommand(maestro.toJsonOutput(result), jsonOptions); + } catch (err) { + await failCommand('maestro', 'Status', err, jsonOptions); + } + }) + .showHelpAfterError(true); + +withFlags( + withFlags( + program + .command('artifacts') + .description( + 'Download reports and/or artifacts (logs, screenshots, video) for a finished Maestro project.', + ) + .requiredOption( + '--id ', + 'Project ID printed when the run started.', + parseProjectId, + ) + .option( + '--report ', + 'Download test report: html, html-detailed, or junit.', + (val) => val.toLowerCase() as ReportFormat, + ) + .option( + '--report-output-dir ', + 'Directory to save test reports (required when --report is used).', + ) + .option( + '--download-artifacts [mode]', + 'Download test artifacts. Mode: all (default) or failed.', + (val) => (val === 'failed' ? 'failed' : 'all') as ArtifactDownloadMode, + ) + .option( + '--artifacts-output-dir ', + 'Directory to save artifacts zip (defaults to current directory).', + ) + .option( + '-q, --quiet', + 'Quieter console output without progress updates.', + ), + JSON_FLAGS, + ), + AUTH_FLAGS, +) + .action(async (args) => { + let jsonOptions: JsonOutputOptions | undefined; + try { + jsonOptions = jsonOptionsFrom(args); + const credentials = await requireCredentials(args); + if (args.debug) enableDebugLogging(); + const maestro = new Maestro( + credentials, + MaestroOptions.forExistingProject({ + quiet: args.quiet || jsonOptions.json || jsonOptions.jsonFile, + report: args.report, + reportOutputDir: args.reportOutputDir, + downloadArtifacts: + args.downloadArtifacts === true + ? 'all' + : (args.downloadArtifacts as ArtifactDownloadMode | undefined), + artifactsOutputDir: args.artifactsOutputDir, + debug: args.debug, + }), + ); + const result = await maestro.artifacts(args.id); + await finishCommand(maestro.toJsonOutput(result), jsonOptions); + } catch (err) { + await failCommand('maestro', 'Artifacts', err, jsonOptions); + } + }) + .showHelpAfterError(true); + +withFlags( + withFlags( + program + .command('list') + .description( + 'List recent Maestro projects on your account, newest first.', + ) + .option( + '--count ', + 'Maximum number of projects to return (default 10).', + (val) => parseInt(val, 10), + ) + .option( + '--offset ', + 'Number of projects to skip, for pagination (default 0).', + (val) => parseInt(val, 10), + ), + JSON_FLAGS, + ), + AUTH_FLAGS, +) + .action(async (args) => { + let jsonOptions: JsonOutputOptions | undefined; + try { + jsonOptions = jsonOptionsFrom(args); + const credentials = await requireCredentials(args); + if (args.debug) enableDebugLogging(); + const maestro = new Maestro( + credentials, + MaestroOptions.forExistingProject({ quiet: true, debug: args.debug }), + ); + const page = await maestro.listProjects({ + count: args.count, + offset: args.offset, + }); + const projects = page.data.map((project) => ({ + id: project.id, + name: project.name, + completed: project.completed, + createdAt: project.created_at, + runs: project.runs, + flows: (project.flows ?? []).map((flow) => flow.name), + bundleId: project.app?.bundle_id ?? undefined, + appVersion: project.app?.app_version ?? undefined, + url: `https://testingbot.com/members/maestro/${project.id}`, + })); + const output = { + provider: 'maestro' as const, + meta: page.meta, + projects, + }; + const written = await writeJsonOutput(output, jsonOptions); + if (!jsonOptions.json) { + printProjectList(projects, page.meta); + if (written) logger.info(`JSON results written to ${written}`); + } + process.exitCode = 0; + } catch (err) { + await failCommand('maestro', 'List', err, jsonOptions); + } + }) + .showHelpAfterError(true); + +function printProjectList( + projects: Array<{ + id: number; + name: string; + completed: boolean; + createdAt: string; + runs: number[]; + flows: string[]; + url: string; + }>, + meta: { offset: number; count: number; total: number }, +): void { + if (projects.length === 0) { + console.log('No Maestro projects found.'); + return; + } + const idWidth = Math.max(2, ...projects.map((p) => String(p.id).length)); + const nameWidth = Math.min( + 40, + Math.max(4, ...projects.map((p) => p.name.length)), + ); + const header = `${'ID'.padEnd(idWidth)} ${'NAME'.padEnd(nameWidth)} ${'STATE'.padEnd(9)} ${'RUNS'.padEnd(4)} ${'FLOWS'.padEnd(5)} CREATED`; + console.log(header); + console.log('-'.repeat(header.length)); + for (const p of projects) { + const name = + p.name.length > nameWidth ? `${p.name.slice(0, nameWidth - 1)}…` : p.name; + console.log( + `${String(p.id).padEnd(idWidth)} ${name.padEnd(nameWidth)} ${(p.completed ? 'completed' : 'running').padEnd(9)} ${String(p.runs.length).padEnd(4)} ${String(p.flows.length).padEnd(5)} ${p.createdAt}`, + ); + } + const shownTo = meta.offset + projects.length; + console.log( + `\nShowing ${meta.offset + 1}-${shownTo} of ${meta.total}. Use --offset ${shownTo} for the next page.`, + ); +} + program .command('login') .description('Authenticate with TestingBot via browser.') diff --git a/src/models/maestro_options.ts b/src/models/maestro_options.ts index 895e1cb..1e72faf 100644 --- a/src/models/maestro_options.ts +++ b/src/models/maestro_options.ts @@ -52,6 +52,22 @@ export default class MaestroOptions { return app?.toLowerCase().endsWith('.ipa') ?? false; } + /** + * Options for commands that act on an already-created project (`status`, + * `artifacts`): no app or flows are uploaded, so only output-related + * settings apply. + */ + public static forExistingProject(options: { + quiet?: boolean; + report?: ReportFormat; + reportOutputDir?: string; + downloadArtifacts?: ArtifactDownloadMode; + artifactsOutputDir?: string; + debug?: boolean; + }): MaestroOptions { + return new MaestroOptions('', [], undefined, options); + } + private _app: string; private _flows: string[]; private _otherApps: string[]; diff --git a/src/providers/maestro.ts b/src/providers/maestro.ts index 4c080e7..f3f06a9 100644 --- a/src/providers/maestro.ts +++ b/src/providers/maestro.ts @@ -92,6 +92,33 @@ export interface MaestroStatusResponse { export type MaestroResult = ProviderResult; +/** One entry of GET /app-automate/maestro (project list). */ +export interface MaestroProjectSummary { + id: number; + name: string; + created_at: string; + updated_at: string; + completed: boolean; + app?: { + app_url?: string; + icon_url?: string | null; + app_version?: string | null; + bundle_id?: string | null; + }; + flows?: { id: number; name: string }[]; + runs: number[]; +} + +export interface MaestroProjectListResponse { + data: MaestroProjectSummary[]; + meta: { offset: number; count: number; total: number }; +} + +export interface ListProjectsParams { + count?: number; + offset?: number; +} + export interface MaestroSocketMessage { id: number; payload: string; @@ -2632,6 +2659,172 @@ export default class Maestro extends BaseProvider { }; } + /** + * `testingbot status`: reports the current state of an existing project. + * With `wait`, blocks until every run has finished and prints the same live + * table and summary as a foreground `maestro` run. + */ + public async status( + appId: number, + options: { wait?: boolean } = {}, + ): Promise { + this.appId = appId; + try { + if (options.wait) { + this.setupSignalHandlers(); + try { + if (!this.options.quiet) { + logger.info(`Waiting for project ${appId} to complete...`); + } + return await this.waitForCompletion(); + } finally { + this.removeSignalHandlers(); + } + } + + const status = await this.getStatus(); + if (!this.options.quiet) { + this.printStatusSummary(status); + } + const outcome = !status.completed + ? 'running' + : this.computeOverallSuccess(status.runs) + ? 'passed' + : 'failed'; + return { success: outcome === 'passed', outcome, runs: status.runs }; + } catch (error) { + this.spinner.stop(); + this.stopFlowAnimation(); + return this.errorResult(error); + } + } + + /** + * `testingbot artifacts`: downloads reports and/or artifact bundles for a + * finished project, reusing the same code path as `--report` and + * `--download-artifacts` on a foreground run. + */ + public async artifacts(appId: number): Promise { + this.appId = appId; + try { + if (!this.options.report && !this.options.downloadArtifacts) { + throw new TestingBotError( + 'Nothing to download: pass --report and/or --download-artifacts.', + ); + } + if (this.options.report && !this.options.reportOutputDir) { + throw new TestingBotError( + '--report-output-dir is required when --report is specified', + ); + } + if (this.options.reportOutputDir) { + await this.ensureOutputDirectory(this.options.reportOutputDir); + } + if (this.options.downloadArtifacts && this.options.artifactsOutputDir) { + await this.ensureOutputDirectory(this.options.artifactsOutputDir); + } + + const status = await this.getStatus(); + if (!status.completed) { + throw new TestingBotError( + `Project ${appId} is still running. Wait for it with "testingbot status --id ${appId} --wait" and try again.`, + ); + } + + await this.fetchReports(status.runs); + await this.downloadArtifacts(status.runs); + + const allSucceeded = this.computeOverallSuccess(status.runs); + return { + success: allSucceeded, + outcome: allSucceeded ? 'passed' : 'failed', + runs: status.runs, + }; + } catch (error) { + this.spinner.stop(); + return this.errorResult(error); + } + } + + /** `testingbot list`: newest-first page of the account's Maestro projects. */ + public async listProjects( + params: ListProjectsParams = {}, + ): Promise { + try { + return await this.withRetry('Listing Maestro projects', async () => { + const response = await axios.get(this.URL, { + params: { + ...(params.count != null && { count: params.count }), + ...(params.offset != null && { offset: params.offset }), + }, + headers: { 'User-Agent': utils.getUserAgent() }, + auth: { + username: this.credentials.userName, + password: this.credentials.accessKey, + }, + }); + if (this.options.debug) { + logger.debug(`Project list: ${JSON.stringify(response.data)}`); + } + return response.data; + }); + } catch (error) { + throw await this.handleErrorWithDiagnostics( + error, + 'Failed to list Maestro projects', + ); + } + } + + /** Logs the error the way run() does and returns an `error` result. */ + private errorResult(error: unknown): MaestroResult { + logger.error(error instanceof Error ? error.message : String(error)); + if (error instanceof Error && error.cause) { + const causeMessage = this.extractErrorMessage(error.cause); + if (causeMessage) { + logger.error(` Reason: ${causeMessage}`); + } + } + return { + success: false, + outcome: 'error', + error: error instanceof Error ? error.message : String(error), + runs: [], + }; + } + + /** One-shot, non-animated snapshot of a project for `testingbot status`. */ + private printStatusSummary(status: MaestroStatusResponse): void { + const url = this.dashboardUrl(); + console.log( + ` Project ${this.appId}: ${status.completed ? 'completed' : pc.cyan('running')}${url ? pc.dim(` ${url}`) : ''}`, + ); + for (const run of status.runs) { + const info = this.getStatusInfo(run.status); + const verdict = + run.status === 'DONE' || run.status === 'FAILED' + ? this.runPassed(run) + ? pc.green('passed') + : pc.red('failed') + : info.text; + console.log( + ` ${info.symbol} Run ${run.id} ${pc.dim(`(${this.getRunDisplayName(run)})`)}: ${verdict}`, + ); + const flows = run.flows ?? []; + this.flowAttempts = this.computeFlowAttempts(flows); + for (const flow of flows.slice().sort((a, b) => a.id - b.id)) { + const display = this.getFlowStatusDisplay(flow); + const errors = + flow.error_messages && flow.error_messages.length > 0 + ? pc.red(` ${flow.error_messages[0]}`) + : ''; + console.log( + ` ${display.colored} ${this.colorizeRetryIcon(this.flowRowName(flow))}${errors}`, + ); + } + } + } + private displayRunStatus( runs: MaestroRunInfo[], startTime: number, diff --git a/src/utils/json_output.ts b/src/utils/json_output.ts index 017fcd2..75032a9 100644 --- a/src/utils/json_output.ts +++ b/src/utils/json_output.ts @@ -17,10 +17,17 @@ export const EXIT_TEST_FAILURE = 2; * How a command ended. * - `passed`/`failed`: tests ran to completion. * - `started`: --async, tests were submitted but not awaited. + * - `running`: `status` was queried while the project is still executing. * - `dry-run`: nothing was sent to the API. * - `error`: the CLI or the infrastructure failed before a verdict. */ -export type RunOutcome = 'passed' | 'failed' | 'started' | 'dry-run' | 'error'; +export type RunOutcome = + | 'passed' + | 'failed' + | 'started' + | 'running' + | 'dry-run' + | 'error'; export interface JsonFlowResult { id: number; @@ -82,7 +89,7 @@ export function resolveExitCode( options: JsonOutputOptions, ): number { if (output.outcome === 'error') return EXIT_ERROR; - if (output.success) return EXIT_SUCCESS; + if (output.outcome !== 'failed') return EXIT_SUCCESS; return options.jsonFile ? EXIT_SUCCESS : EXIT_TEST_FAILURE; } @@ -97,7 +104,10 @@ export function validateJsonOptions(options: JsonOutputOptions): void { * Default file name for --json-file: `_testingbot.json`, or a stable * name when the command failed before an app id was assigned. */ -export function defaultJsonFileName(output: JsonOutput): string { +export function defaultJsonFileName(output: { + appId?: number; + provider: string; +}): string { return output.appId ? `${output.appId}_testingbot.json` : `${output.provider}_testingbot.json`; @@ -107,10 +117,9 @@ export function defaultJsonFileName(output: JsonOutput): string { * Writes the JSON document to stdout (--json) and/or a file (--json-file). * Returns the path written, if any. */ -export async function writeJsonOutput( - output: JsonOutput, - options: JsonOutputOptions, -): Promise { +export async function writeJsonOutput< + T extends { provider: string; appId?: number }, +>(output: T, options: JsonOutputOptions): Promise { const serialized = JSON.stringify(output, null, 2); if (options.json) { diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 971f1d8..bafb18d 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -1082,6 +1082,281 @@ describe('TestingBotCTL CLI', () => { }); }); + describe('status, artifacts and list commands', () => { + let mockStatus: jest.Mock; + let mockArtifacts: jest.Mock; + let mockListProjects: jest.Mock; + let stdoutSpy: jest.SpyInstance; + let consoleSpy: jest.SpyInstance; + + beforeEach(() => { + mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' }); + mockStatus = jest.fn(); + mockArtifacts = jest.fn(); + mockListProjects = jest.fn(); + Maestro.prototype.status = mockStatus; + Maestro.prototype.artifacts = mockArtifacts; + Maestro.prototype.listProjects = mockListProjects; + stdoutSpy = jest + .spyOn(process.stdout, 'write') + .mockImplementation(() => true); + consoleSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + test('status queries the project and exits 0 when it passed', async () => { + mockStatus.mockResolvedValue({ + success: true, + outcome: 'passed', + runs: [], + }); + await program.parseAsync(['node', 'cli', 'status', '--id', '1234']); + expect(mockStatus).toHaveBeenCalledWith(1234, { wait: undefined }); + expect(process.exitCode).toBe(0); + }); + + test('status exits 2 when the project failed and 0 while still running', async () => { + mockStatus.mockResolvedValue({ + success: false, + outcome: 'failed', + runs: [], + }); + await program.parseAsync(['node', 'cli', 'status', '--id', '1234']); + expect(process.exitCode).toBe(2); + + process.exitCode = 0; + mockStatus.mockResolvedValue({ + success: false, + outcome: 'running', + runs: [], + }); + await program.parseAsync(['node', 'cli', 'status', '--id', '1234']); + expect(process.exitCode).toBe(0); + }); + + test('status --wait passes wait and forwards quiet', async () => { + mockStatus.mockResolvedValue({ + success: true, + outcome: 'passed', + runs: [], + }); + await program.parseAsync([ + 'node', + 'cli', + 'status', + '--id', + '1234', + '--wait', + '--quiet', + ]); + expect(mockStatus).toHaveBeenCalledWith(1234, { wait: true }); + expect(lastConstructorOptions<{ quiet: boolean }>(Maestro).quiet).toBe( + true, + ); + }); + + test('status --json prints the document and moves logs off stdout', async () => { + mockStatus.mockResolvedValue({ + success: false, + outcome: 'running', + runs: [], + }); + await program.parseAsync([ + 'node', + 'cli', + 'status', + '--id', + '1234', + '--json', + ]); + expect(JSON.parse(String(stdoutSpy.mock.calls[0][0]))).toMatchObject({ + provider: 'maestro', + outcome: 'running', + }); + expect(lastConstructorOptions<{ quiet: boolean }>(Maestro).quiet).toBe( + true, + ); + expect(process.exitCode).toBe(0); + }); + + test('status rejects a non-numeric project id with usage help', async () => { + const stderrSpy = jest + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + await expect( + program.parseAsync(['node', 'cli', 'status', '--id', 'abc']), + ).rejects.toThrow('process.exit called with code: 1'); + expect(mockStatus).not.toHaveBeenCalled(); + expect(stderrSpy.mock.calls.map((c) => String(c[0])).join('')).toContain( + 'expected a positive integer', + ); + }); + + test('status requires --id', async () => { + await expect( + program.parseAsync(['node', 'cli', 'status']), + ).rejects.toThrow('process.exit called with code: 1'); + expect(mockStatus).not.toHaveBeenCalled(); + }); + + test('status exits 1 without credentials', async () => { + mockGetCredentials.mockResolvedValue(null); + await program.parseAsync(['node', 'cli', 'status', '--id', '1234']); + expect(mockStatus).not.toHaveBeenCalled(); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('No TestingBot credentials found'), + ); + expect(process.exitCode).toBe(1); + }); + + test('artifacts forwards report and artifact options', async () => { + mockArtifacts.mockResolvedValue({ + success: true, + outcome: 'passed', + runs: [], + }); + await program.parseAsync([ + 'node', + 'cli', + 'artifacts', + '--id', + '1234', + '--report', + 'JUNIT', + '--report-output-dir', + './reports', + '--download-artifacts', + 'failed', + '--artifacts-output-dir', + './out', + ]); + expect(mockArtifacts).toHaveBeenCalledWith(1234); + const opts = lastConstructorOptions<{ + report?: string; + reportOutputDir?: string; + downloadArtifacts?: string; + artifactsOutputDir?: string; + }>(Maestro); + expect(opts.report).toBe('junit'); + expect(opts.reportOutputDir).toBe('./reports'); + expect(opts.downloadArtifacts).toBe('failed'); + expect(opts.artifactsOutputDir).toBe('./out'); + expect(process.exitCode).toBe(0); + }); + + test('artifacts defaults --download-artifacts to all and exits 1 on error', async () => { + mockArtifacts.mockResolvedValue({ + success: false, + outcome: 'error', + error: 'still running', + runs: [], + }); + await program.parseAsync([ + 'node', + 'cli', + 'artifacts', + '--id', + '1234', + '--download-artifacts', + ]); + expect( + lastConstructorOptions<{ downloadArtifacts?: string }>(Maestro) + .downloadArtifacts, + ).toBe('all'); + expect(process.exitCode).toBe(1); + }); + + test('list prints a table and passes pagination through', async () => { + mockListProjects.mockResolvedValue({ + data: [ + { + id: 42, + name: 'nightly', + created_at: '2026-09-01T10:00:00Z', + updated_at: '2026-09-01T10:05:00Z', + completed: true, + app: { bundle_id: 'com.example', app_version: '1.0' }, + flows: [{ id: 1, name: 'login' }], + runs: [7, 8], + }, + ], + meta: { offset: 5, count: 1, total: 20 }, + }); + await program.parseAsync([ + 'node', + 'cli', + 'list', + '--count', + '1', + '--offset', + '5', + ]); + expect(mockListProjects).toHaveBeenCalledWith({ count: 1, offset: 5 }); + const printed = consoleSpy.mock.calls.map((c) => String(c[0])).join('\n'); + expect(printed).toContain('42'); + expect(printed).toContain('nightly'); + expect(printed).toContain('completed'); + expect(printed).toContain('Showing 6-6 of 20'); + expect(lastConstructorOptions<{ quiet: boolean }>(Maestro).quiet).toBe( + true, + ); + expect(process.exitCode).toBe(0); + }); + + test('list --json emits projects with dashboard urls', async () => { + mockListProjects.mockResolvedValue({ + data: [ + { + id: 42, + name: 'nightly', + created_at: '2026-09-01T10:00:00Z', + updated_at: '2026-09-01T10:05:00Z', + completed: false, + flows: [], + runs: [], + }, + ], + meta: { offset: 0, count: 10, total: 1 }, + }); + await program.parseAsync(['node', 'cli', 'list', '--json']); + expect(mockListProjects).toHaveBeenCalledWith({ + count: undefined, + offset: undefined, + }); + const output = JSON.parse(String(stdoutSpy.mock.calls[0][0])); + expect(output).toEqual({ + provider: 'maestro', + meta: { offset: 0, count: 10, total: 1 }, + projects: [ + { + id: 42, + name: 'nightly', + completed: false, + createdAt: '2026-09-01T10:00:00Z', + runs: [], + flows: [], + url: 'https://testingbot.com/members/maestro/42', + }, + ], + }); + expect(consoleSpy).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(0); + }); + + test('list exits 1 and emits an error document when the API fails', async () => { + mockListProjects.mockRejectedValue(new Error('boom')); + await program.parseAsync(['node', 'cli', 'list', '--json']); + expect(JSON.parse(String(stdoutSpy.mock.calls[0][0]))).toMatchObject({ + outcome: 'error', + error: 'boom', + }); + expect(process.exitCode).toBe(1); + }); + }); + test('unknown command should show help', async () => { const exitSpy = jest .spyOn(process, 'exit') diff --git a/tests/providers/maestro.test.ts b/tests/providers/maestro.test.ts index 2440c1b..17d8970 100644 --- a/tests/providers/maestro.test.ts +++ b/tests/providers/maestro.test.ts @@ -6827,4 +6827,206 @@ onFlowStart: expect(result.error).toBe('Upload failed'); }); }); + + describe('status(), artifacts() and listProjects()', () => { + const run = (overrides: Record = {}) => ({ + id: 5678, + status: 'DONE', + capabilities: { deviceName: 'Pixel 6', platformName: 'Android' }, + success: 1, + flows: [{ id: 1, name: 'login', status: 'DONE', success: 1 }], + ...overrides, + }); + + beforeEach(() => { + jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + it('status() reports passed for a completed, green project', async () => { + maestro['getStatus'] = jest + .fn() + .mockResolvedValue({ runs: [run()], success: true, completed: true }); + const result = await maestro.status(1234); + expect(maestro['appId']).toBe(1234); + expect(result.outcome).toBe('passed'); + expect(result.success).toBe(true); + expect(result.runs).toHaveLength(1); + expect(console.log).toHaveBeenCalledWith( + expect.stringContaining('Project 1234: completed'), + ); + expect(console.log).toHaveBeenCalledWith( + expect.stringContaining('login'), + ); + }); + + it('status() reports failed using last-attempt-wins', async () => { + const flows = [ + { id: 1, name: 'login', status: 'DONE', success: 1 }, + { + id: 2, + name: 'login', + status: 'FAILED', + success: 0, + error_messages: ['x'], + }, + ]; + maestro['getStatus'] = jest.fn().mockResolvedValue({ + runs: [run({ success: 0, flows })], + success: false, + completed: true, + }); + const result = await maestro.status(1234); + expect(result.outcome).toBe('failed'); + expect(result.success).toBe(false); + }); + + it('status() reports running while the project is incomplete', async () => { + maestro['getStatus'] = jest.fn().mockResolvedValue({ + runs: [run({ status: 'READY', success: 0 })], + success: false, + completed: false, + }); + const result = await maestro.status(1234); + expect(result.outcome).toBe('running'); + expect(result.success).toBe(false); + }); + + it('status() stays silent in quiet mode', async () => { + const quiet = new Maestro( + mockCredentials, + MaestroOptions.forExistingProject({ quiet: true }), + ); + quiet['getStatus'] = jest + .fn() + .mockResolvedValue({ runs: [run()], success: true, completed: true }); + await quiet.status(1234); + expect(console.log).not.toHaveBeenCalled(); + }); + + it('status({ wait: true }) polls to completion via waitForCompletion', async () => { + maestro['waitForCompletion'] = jest.fn().mockResolvedValue({ + success: true, + outcome: 'passed', + runs: [run()], + }); + maestro['getStatus'] = jest.fn(); + const result = await maestro.status(1234, { wait: true }); + expect(maestro['waitForCompletion']).toHaveBeenCalledTimes(1); + expect(maestro['getStatus']).not.toHaveBeenCalled(); + expect(result.outcome).toBe('passed'); + }); + + it('status() returns an error result when the API call fails', async () => { + maestro['getStatus'] = jest + .fn() + .mockRejectedValue(new TestingBotError('Nothing found for 1234')); + const result = await maestro.status(1234); + expect(result).toEqual({ + success: false, + outcome: 'error', + error: 'Nothing found for 1234', + runs: [], + }); + }); + + it('artifacts() rejects when nothing was requested', async () => { + const result = await maestro.artifacts(1234); + expect(result.outcome).toBe('error'); + expect(result.error).toContain('Nothing to download'); + }); + + it('artifacts() requires --report-output-dir with --report', async () => { + const m = new Maestro( + mockCredentials, + MaestroOptions.forExistingProject({ report: 'junit' }), + ); + const result = await m.artifacts(1234); + expect(result.outcome).toBe('error'); + expect(result.error).toContain('--report-output-dir is required'); + }); + + it('artifacts() refuses a project that is still running', async () => { + const m = new Maestro( + mockCredentials, + MaestroOptions.forExistingProject({ downloadArtifacts: 'all' }), + ); + m['getStatus'] = jest.fn().mockResolvedValue({ + runs: [run({ status: 'READY' })], + success: false, + completed: false, + }); + m['downloadArtifacts'] = jest.fn(); + const result = await m.artifacts(1234); + expect(result.outcome).toBe('error'); + expect(result.error).toContain('testingbot status --id 1234 --wait'); + expect(m['downloadArtifacts']).not.toHaveBeenCalled(); + }); + + it('artifacts() downloads reports and artifacts for a finished project', async () => { + const m = new Maestro( + mockCredentials, + MaestroOptions.forExistingProject({ + report: 'junit', + reportOutputDir: './reports', + downloadArtifacts: 'failed', + artifactsOutputDir: './out', + }), + ); + m['ensureOutputDirectory'] = jest.fn().mockResolvedValue(undefined); + const runs = [ + run({ + success: 0, + flows: [{ id: 1, name: 'login', status: 'FAILED', success: 0 }], + }), + ]; + m['getStatus'] = jest + .fn() + .mockResolvedValue({ runs, success: false, completed: true }); + m['fetchReports'] = jest.fn().mockResolvedValue(undefined); + m['downloadArtifacts'] = jest.fn().mockResolvedValue(undefined); + + const result = await m.artifacts(1234); + + expect(m['ensureOutputDirectory']).toHaveBeenCalledWith('./reports'); + expect(m['ensureOutputDirectory']).toHaveBeenCalledWith('./out'); + expect(m['fetchReports']).toHaveBeenCalledWith(runs); + expect(m['downloadArtifacts']).toHaveBeenCalledWith(runs); + expect(result.outcome).toBe('failed'); + expect(result.runs).toBe(runs); + }); + + it('listProjects() calls the project list endpoint with pagination', async () => { + const page = { + data: [ + { + id: 1, + name: 'p', + created_at: '', + updated_at: '', + completed: true, + runs: [], + }, + ], + meta: { offset: 5, count: 2, total: 9 }, + }; + axios.get = jest.fn().mockResolvedValue({ data: page }); + const result = await maestro.listProjects({ count: 2, offset: 5 }); + expect(axios.get).toHaveBeenCalledWith( + 'https://api.testingbot.com/v1/app-automate/maestro', + expect.objectContaining({ + params: { count: 2, offset: 5 }, + auth: { username: 'testUser', password: 'testKey' }, + }), + ); + expect(result).toBe(page); + }); + + it('listProjects() omits unset pagination params', async () => { + axios.get = jest.fn().mockResolvedValue({ + data: { data: [], meta: { offset: 0, count: 10, total: 0 } }, + }); + await maestro.listProjects(); + expect((axios.get as jest.Mock).mock.calls[0][1].params).toEqual({}); + }); + }); });